feat: working data download and indexing

bin/index.js

@@ -0,0 +1,23 @@
+/* eslint no-global-assign: 0 no-console: 0 */
+require = require('esm')(module);
+const getopts = require('getopts');
+const mod = require('../src/index.mjs').default;
+
+const { index, ...elasticsearch } = getopts(process.argv.slice(2), {
+  string: ['host', 'log'],
+  alias: {
+    h: 'host',
+    l: 'log',
+    i: 'index',
+  },
+  default: {
+    host: 'localhost:9200',
+    log: 'error',
+    index: 'adsb-data',
+  },
+});
+
+mod(index, { elasticsearch }).catch(err => {
+  console.error(err);
+  process.exit(1);
+});

index.js

@@ -1,5 +0,0 @@
-/* eslint no-global-assign: 0 */
-require = require('esm')(module);
-const mod = require('./src/index.mjs').default;
-
-module.exports = mod;

index.mjs

@@ -1,3 +0,0 @@
-import mod from './src/index.mjs';
-
-export default mod;

package.json

@@ -1,14 +1,13 @@
 {
   "name": "adsb-index",
   "version": "0.0.0",
+  "private": true,
   "description": "ADS-B indexing script",
-  "main": "index",
+  "bin": "bin/index.js",
-  "module": "index.mjs",
   "scripts": {
-    "lint": "eslint '*.{js,mjs}' 'src/**/*.{js,mjs}'",
+    "lint": "eslint '*.{js,mjs}' 'src/**/*.{js,mjs}' 'bin/**/*.{js,mjs}'",
     "precommit": "lint-staged",
-    "version": "npm-auto-version",
+    "version": "npm-auto-version"
-    "start": "node ."
   },
   "repository": {
     "type": "git",
@@ -50,19 +49,23 @@
     "cjs": true
   },
   "dependencies": {
-    "esm": "^3.0.17"
+    "axios": "^0.18.0",
+    "elasticsearch": "^15.2.0",
+    "esm": "^3.0.17",
+    "getopts": "^2.2.2",
+    "node-cron": "^2.0.3"
   },
   "devDependencies": {
     "@w33ble/npm-auto-tools": "*",
     "eslint": "^4.9.0",
+    "eslint-config-airbnb": "^16.1.0",
     "eslint-config-prettier": "^2.9.0",
-    "eslint-plugin-prettier": "^2.3.1",
     "eslint-plugin-import": "^2.7.0",
+    "eslint-plugin-jsx-a11y": "^6.0.2",
+    "eslint-plugin-prettier": "^2.3.1",
+    "eslint-plugin-react": "^7.1.0",
     "husky": "^0.14.3",
     "lint-staged": "^7.0.4",
-    "eslint-config-airbnb": "^16.1.0",
-    "eslint-plugin-jsx-a11y": "^6.0.2",
-    "eslint-plugin-react": "^7.1.0",
     "prettier": "^1.9.0"
   }
 }

src/index.mjs

@@ -1,3 +1,43 @@
-export default function() {
+import elasticsearch from 'elasticsearch';
-  // es6 module code goes here
+import logger from './lib/logger.mjs';
+import { createIndex, bulkInsert } from './lib/data-source.mjs';
+import getData from './lib/get-data.mjs';
+
+const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT'];
+
+export default async function(indexName, opts = {}) {
+  const client = new elasticsearch.Client({
+    host: opts.elasticsearch.host,
+    log: opts.elasticsearch.log,
+  });
+
+  // make sure we can connect to the node
+  await client.ping();
+
+  // create the target index
+  const index = await createIndex(client, indexName);
+
+  const records = await getData();
+  logger.debug(`ADS-B records:, ${records.length}`);
+
+  // index all the data
+  const documents = records.map(rec => ({
+    transponder: rec[0],
+    callsign: rec[1],
+    origin_country: rec[2],
+    time_position: rec[3],
+    last_contact: rec[4],
+    location: rec[5] && rec[6] ? `${rec[6]},${rec[5]}` : null,
+    baro_altitude: rec[7],
+    geo_altitude: rec[13],
+    on_ground: rec[8],
+    velocity: rec[9],
+    vertical_rate: rec[11],
+    squawk: rec[14],
+    spi: rec[15],
+    position_source: positionSourceMap[rec[16]],
+  }));
+
+  await bulkInsert(client, index, documents);
+  logger.debug(`Successfully indexed ${records.length} records to ${index}`);
 }

src/lib/data-source.mjs

@@ -0,0 +1,80 @@
+import logger from './logger.mjs';
+import { BadRequest } from './es-errors.mjs';
+import zeroPad from './zero-pad.mjs';
+
+const doctype = 'doc';
+
+// helper for time-based indices
+function getIndexName(index) {
+  const d = new Date();
+  const [year, month, day] = [
+    d.getFullYear(),
+    zeroPad(d.getMonth() + 1, 2),
+    zeroPad(d.getDate(), 2),
+  ];
+  return `${index}-${year}.${month}.${day}`;
+}
+
+export async function createIndex(client, index) {
+  const realIndex = getIndexName(index);
+  return client.indices
+    .create({
+      index: realIndex,
+      body: {
+        settings: {},
+        mappings: {
+          [doctype]: {
+            properties: {
+              transponder: { type: 'keyword' },
+              callsign: { type: 'keyword' },
+              origin_country: { type: 'keyword' },
+              time_position: { type: 'date' },
+              last_contact: { type: 'date' },
+              location: { type: 'geo_point' },
+              baro_altitude: { type: 'float' },
+              geo_altitude: { type: 'float' },
+              on_ground: { type: 'boolean' },
+              velocity: { type: 'float' },
+              vertical_rate: { type: 'float' },
+              squawk: { type: 'keyword' },
+              spi: { type: 'boolean' },
+              position_source: { type: 'keyword' },
+            },
+          },
+        },
+      },
+    })
+    .then(res => {
+      logger.debug(`Index created: ${res.index}`);
+      return res.index;
+    })
+    .catch(err => {
+      // check for existing index
+      if (err instanceof BadRequest) {
+        logger.debug(`Index exists: ${realIndex}`);
+        return client.indices.get({ index: realIndex }).then(() => realIndex);
+      }
+
+      throw err;
+    });
+}
+
+export async function createDocument(client, index, body) {
+  client.index({
+    index,
+    type: doctype,
+    body,
+  });
+}
+
+export async function bulkInsert(client, index, docs) {
+  const body = docs.reduce((collection, doc) => {
+    collection.push({ index: { _index: index, _type: doctype } });
+    collection.push(doc);
+    return collection;
+  }, []);
+
+  client.bulk({
+    body,
+  });
+}

src/lib/es-errors.mjs

@@ -0,0 +1,3 @@
+import elasticsearch from 'elasticsearch';
+
+export const { BadRequest, NotFound, Forbidden, Conflict } = elasticsearch.errors;

src/lib/fetch.mjs

@@ -0,0 +1,13 @@
+import axios from 'axios';
+
+const api = 'https://opensky-network.org/api';
+
+export default axios.create({
+  baseURL: api,
+  timeout: 3000,
+  responseType: 'json',
+  headers: {
+    'X-Custom-Header': 'foobar',
+    'Content-Type': 'application/json; charset=utf-8',
+  },
+});

src/lib/get-data.mjs

@@ -0,0 +1,6 @@
+import fetch from './fetch.mjs';
+
+export default async function getData() {
+  const res = await fetch.get('/states/all');
+  return res.data.states;
+}

src/lib/logger.mjs

@@ -0,0 +1,35 @@
+/* eslint no-console: 0 */
+import zeroPad from './zero-pad.mjs';
+
+const getTimestamp = () => {
+  const d = new Date();
+  return `${[d.getFullYear(), zeroPad(d.getMonth() + 1, 2), zeroPad(d.getDate(), 2)].join('/')} ${[
+    zeroPad(d.getHours(), 2),
+    zeroPad(d.getMinutes(), 2),
+    zeroPad(d.getSeconds(), 2),
+  ].join(':')}`;
+};
+
+const wrapMessage = msg => `[${getTimestamp()}]: ${msg.map(m => JSON.stringify(m)).join(' ')}`;
+
+const logger = {
+  log(...args) {
+    console.log(wrapMessage(args));
+  },
+  warn(...args) {
+    console.warn(wrapMessage(args));
+  },
+  error(...args) {
+    if (args[0] instanceof Error) {
+      console.error(getTimestamp(), args[0].stack);
+      return;
+    }
+    console.error(wrapMessage(args));
+  },
+  debug(...args) {
+    if (!process.env.DEBUG) return;
+    console.log(wrapMessage(args));
+  },
+};
+
+export default logger;

src/lib/zero-pad.mjs

@@ -0,0 +1,9 @@
+export default function zeroPad(str, len) {
+  let output = `${str}`;
+  if (!len || len <= output.length) return str;
+
+  while (output.length < len) {
+    output = `0${output}`;
+  }
+  return output;
+}

yarn.lock

@@ -36,6 +36,13 @@ acorn@^5.5.0:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.3.tgz#67aa231bf8812974b85235a96771eb6bd07ea279"
   integrity sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==
 
+agentkeepalive@^3.4.1:
+  version "3.5.2"
+  resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-3.5.2.tgz#a113924dd3fa24a0bc3b78108c450c2abee00f67"
+  integrity sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==
+  dependencies:
+    humanize-ms "^1.2.1"
+
 ajv-keywords@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-2.1.1.tgz#617997fc5f60576894c435f940d819e135b80762"
@@ -193,6 +200,14 @@ auto-changelog@^1.7.0:
     parse-github-url "^1.0.1"
     semver "^5.1.0"
 
+axios@^0.18.0:
+  version "0.18.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102"
+  integrity sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=
+  dependencies:
+    follow-redirects "^1.3.0"
+    is-buffer "^1.1.5"
+
 axobject-query@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.0.2.tgz#ea187abe5b9002b377f925d8bf7d1c561adf38f9"
@@ -496,6 +511,13 @@ date-fns@^1.27.2:
   resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.29.0.tgz#12e609cdcb935127311d04d33334e2960a2a54e6"
   integrity sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==
 
+debug@=3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
+  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
+  dependencies:
+    ms "2.0.0"
+
 debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -582,6 +604,15 @@ doctrine@^2.1.0:
   dependencies:
     esutils "^2.0.2"
 
+elasticsearch@^15.2.0:
+  version "15.2.0"
+  resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-15.2.0.tgz#234362b5aa743d9c0a925566569ea7813b8f2569"
+  integrity sha512-jOFcBoEh3Sn3gjUTozInODZTLriJtfppAUC7jnQCUE+OUj8o7GoAyC+L4h/L3ZxmXNFbQCunqVR+nmSofHdo9A==
+  dependencies:
+    agentkeepalive "^3.4.1"
+    chalk "^1.0.0"
+    lodash "^4.17.10"
+
 elegant-spinner@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e"
@@ -979,6 +1010,13 @@ flat-cache@^1.2.1:
     graceful-fs "^4.1.2"
     write "^0.2.1"
 
+follow-redirects@^1.3.0:
+  version "1.5.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.9.tgz#c9ed9d748b814a39535716e531b9196a845d89c6"
+  integrity sha512-Bh65EZI/RU8nx0wbYF9shkFZlqLP+6WT/5FnA3cE/djNSuKNHJEinGGZgu/cQEkeeb2GdFOgenAmn8qaqYke2w==
+  dependencies:
+    debug "=3.1.0"
+
 for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -1026,7 +1064,7 @@ get-value@^2.0.3, get-value@^2.0.6:
   resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
   integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=
 
-getopts@^2.0.6:
+getopts@^2.0.6, getopts@^2.2.2:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.2.tgz#a6ada98961c3d1f099e88f0698fabe716f8a253c"
   integrity sha512-qX9XmOFViBolzBfWkBKOP/HOO8VSgy6reZMVWo/QvCmwXJLZU5idp4W+L70CM8SPBRBwgrCyU65IKKr40zgizw==
@@ -1136,6 +1174,13 @@ hosted-git-info@^2.1.4:
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047"
   integrity sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==
 
+humanize-ms@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed"
+  integrity sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=
+  dependencies:
+    ms "^2.0.0"
+
 husky@^0.14.3:
   version "0.14.3"
   resolved "https://registry.yarnpkg.com/husky/-/husky-0.14.3.tgz#c69ed74e2d2779769a17ba8399b54ce0b63c12c3"
@@ -1744,7 +1789,7 @@ ms@2.0.0:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
   integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
 
-ms@^2.1.1:
+ms@^2.0.0, ms@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
   integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
@@ -1781,6 +1826,14 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
+node-cron@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-2.0.3.tgz#b9649784d0d6c00758410eef22fa54a10e3f602d"
+  integrity sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==
+  dependencies:
+    opencollective-postinstall "^2.0.0"
+    tz-offset "0.0.1"
+
 node-fetch@^2.1.2, node-fetch@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
@@ -1881,6 +1934,11 @@ onetime@^2.0.0:
   dependencies:
     mimic-fn "^1.0.0"
 
+opencollective-postinstall@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz#798e83e168f7b91949061c2683f762af747f17cc"
+  integrity sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==
+
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
@@ -2571,6 +2629,11 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
+tz-offset@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/tz-offset/-/tz-offset-0.0.1.tgz#fef920257024d3583ed9072a767721a18bdb8a76"
+  integrity sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==
+
 uglify-js@^3.1.4:
   version "3.4.9"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.9.tgz#af02f180c1207d76432e473ed24a28f4a782bae3"