hdt

HDT for Node.js

HDT (Header Dictionary Triples) is a compressed format for RDF triples.
The hdt npm package for Node.js brings fast access to HDT files through C bindings.

Usage

Importing the library

Install the library by adding hdt to your package.json or executing

$ npm install hdt

Then require the library.

const hdt = require('hdt');

Opening and closing an HDT document

Open an HDT document with hdt.fromFile, which takes a filename as argument and returns the HDT document in a promise. Close the document with close.

hdt.fromFile('./test/test.hdt').then(function(hdtDocument) {
  // Don't forget to close the document when you're done
  return hdtDocument.close();
});

Searching for triples matching a pattern

Search for triples with search, which takes subject, predicate, object, and options arguments. Subject, predicate, and object can be IRIs or literals, represented as simple strings. If any of these parameters is null or a variable, it is considered a wildcard. Optionally, an offset and limit can be passed in an options object, selecting only the specified subset.

The promise returns an object with an array of triples, the total number of expected triples for the pattern, and whether the total count is an estimate or exact.

var doc;
hdt.fromFile('./test/test.hdt')
  .then(function(hdtDocument) {
    doc = hdtDocument;
    return doc.searchTriples('http://example.org/s1', null, null, { offset: 0, limit: 10 })
  })
  .then(function(result) {
    console.log('Approximately ' + result.totalCount + ' triples match the pattern.');
    result.triples.forEach(function (triple) { console.log(triple); });
    return doc.close();
  });

Counting triples matching a pattern

Retrieve an estimate of the total number of triples matching a pattern with count, which takes subject, predicate, and object arguments.

var doc;
hdt.fromFile('./test/test.hdt')
  .then(function(hdtDocument) {
    doc = hdtDocument;
    return doc.countTriples('http://example.org/s1', null, null);
  })
  .then(function(result) {
    console.log('Approximately ' + result.totalCount + ' triples match the pattern.');
    return doc.close()
  });

Searching for bindings matching a pattern

Search for bindings with searchBindings, which takes bindingsFactory, subject, predicate, object, and options arguments. Subject, predicate, and object can be IRIs, literals, or variables, represented as RDF/JS terms. If any of these parameters is a variable, it is considered a wildcard. Optionally, an offset and limit can be passed in an options object, selecting only the specified subset.

The promise returns an object with an array of bindings, the total number of expected bindings for the pattern, and whether the total count is an estimate or exact.

If variables are reused across terms, this library will make sure to only return bindings when matches for those variables are equal.

const DF = new (require('rdf-data-factory').DataFactory)();
const BF = new (require('@comunica/utils-bindings-factory').BindingsFactory)(DF);

var doc;
hdt.fromFile('./test/test.hdt')
  .then(function(hdtDocument) {
    doc = hdtDocument;
    return doc.searchBindings(DF.namedNode('http://example.org/s1'), DF.variable('p'), DF.variable('o'), { offset: 0, limit: 10 })
  })
  .then(function(result) {
    console.log('Approximately ' + result.totalCount + ' bindings match the pattern.');
    result.bindings.forEach(function (binding) { console.log(binding.toString()); });
    return doc.close();
  });

Search terms starting with a prefix

Find terms (literals and IRIs) that start with a given prefix.

hdtDocument.searchTerms({ prefix: 'http://example.org/', limit: 100, position: 'object' })
  .then(function(suggestions) {
    console.log('Found ' + suggestions.length + ' suggestions');
    return hdtDocument.close();
  });

Fetching unique predicates for a subject and/or an object

Find all unique predicates for a given subject argument.

hdtDocument.searchTerms({ subject: 'http://example.org/s1' limit: 10, position: 'predicate' })
  .then(function(terms) {
    console.log('Found ' + terms.length + ' unique predicates');
    return hdtDocument.close();
  });

Find all unique predicates for a given object argument.

hdtDocument.searchTerms({ object: 'http://example.org/o1', limit: 10, position: 'predicate' })
  .then(function(terms) {
    console.log('Found ' + terms.length + ' unique predicates');
    return hdtDocument.close();
  });

Find all unique predicates for given subject and object arguments.

hdtDocument.searchTerms({ subject: 'http://example.org/s1', object: 'http://example.org/o1', limit: 10, position: 'predicate' })
  .then(function(terms) {
    console.log('Found ' + terms.length + ' unique predicates');
    return hdtDocument.close();
  });

Searching literals containing a substring

In an HDT file that was generated with an FM index, you can search for literals that contain a certain substring.

var doc;
hdt.fromFile('./test/test.hdt')
  .then(function(hdtDocument) {
    doc = hdtDocument;
    return doc.searchLiterals('b', { offset: 0, limit: 5 });
  })
  .then(function(result) {
    console.log('Approximately ' + result.totalCount + ' literals contain the pattern.');
    result.literals.forEach(function (literal) { console.log(literal); });
    return doc.close();
  });

Reading the header

HDT supports reading the header as string using document.readHeader(). The example below reads the header as string, and parses the header using the N3.js library.

var N3 = require('n3');
var doc;
var parser = N3.Parser();
hdt.fromFile('./test/test.hdt')
  .then(function(hdtDocument) {
    doc = hdtDocument;
    return doc.readHeader();
  })
  .then(function(header) {
    var triples = [];
    return new Promise(function(resolve, reject) {
      parser.parse(header, function(error, triple) {
        if (error) return reject(error);
        if (triple) return triples.push(triple);
        resolve(triples);
      });
    });
  })
  .then(function(triples) {
    console.log('Read triples from header:\n', triples);
  })
  .catch(function(e) {
    console.error(e);
  })

Changing the header

To replace header information of an HDT, use document.changeHeader(header, toFile), that returns an HDT document of the output file. The example below serializes an N3 triples object into an N-Triples string, and stores it in the header.

var N3 = require('n3');
var doc;
var outputFile = './out.hdt';

hdt.fromFile('./test/test.hdt')
 .then(function(hdtDocument) {
   doc = hdtDocument;
   return new Promise(function(resolve, reject) {
     var writer = N3.Writer({format: 'N-Triples'});
     writer.addTriple('http://example.org/cartoons#Tom',
                      'http://www.w3.org/1999/02/22-rdf-syntax-ns#type',
                      'http://example.org/cartoons#Cat');
     writer.end(function(error, triples) {
       if (error) return reject(error);
       resolve(triples);
     });
   });
 })
 .then(function(triples) {
      return doc.changeHeader(triples, outputFile);
  })
  .then(function(createdDocument) {
    return createdDocument.readHeader();
  })
  .then(function(result) {
    console.log('Wrote ' + result + ' to ' + outputFile);
  })
  .catch(function(e) {
    console.error(e);
  });

Standalone utility

The standalone utility hdt allows you to query HDT files from the command line.
To install system-wide, execute:

sudo npm install -g hdt

Specify queries as follows:

hdt dataset.hdt --query '?s ?p ?o' --offset 200 --limit 100 --format turtle

Replace any of the query variables by an IRI or literal to match specific patterns.

Build manually

To build the module from source, follow these instructions:

git clone https://github.com/RubenVerborgh/HDT-Node.git hdt
cd hdt
git submodule init
git submodule update
npm install
npm test

If you make changes to the source, do the following to rebuild:

node-gyp build && npm test

License

The Node.js bindings for HDT are written by Ruben Verborgh.

This code is copyrighted by Ruben Verborgh and released under the GNU Lesser General Public License. It uses the HDT C++ Library, released under the same license.