diff --git a/bin/index.js b/bin/index.js new file mode 100644 index 0000000..f0d4f4f --- /dev/null +++ b/bin/index.js @@ -0,0 +1,23 @@ +/* eslint no-global-assign: 0 no-console: 0 */ +require = require('esm')(module); +const getopts = require('getopts'); +const mod = require('../src/index.mjs').default; + +const { index, ...elasticsearch } = getopts(process.argv.slice(2), { + string: ['host', 'log'], + alias: { + h: 'host', + l: 'log', + i: 'index', + }, + default: { + host: 'localhost:9200', + log: 'error', + index: 'adsb-data', + }, +}); + +mod(index, { elasticsearch }).catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/index.js b/index.js deleted file mode 100644 index 8ee32f7..0000000 --- a/index.js +++ /dev/null @@ -1,5 +0,0 @@ -/* eslint no-global-assign: 0 */ -require = require('esm')(module); -const mod = require('./src/index.mjs').default; - -module.exports = mod; diff --git a/index.mjs b/index.mjs deleted file mode 100644 index 5329a9f..0000000 --- a/index.mjs +++ /dev/null @@ -1,3 +0,0 @@ -import mod from './src/index.mjs'; - -export default mod; diff --git a/package.json b/package.json index 6134b88..79792d9 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,13 @@ { "name": "adsb-index", "version": "0.0.0", + "private": true, "description": "ADS-B indexing script", - "main": "index", - "module": "index.mjs", + "bin": "bin/index.js", "scripts": { - "lint": "eslint '*.{js,mjs}' 'src/**/*.{js,mjs}'", + "lint": "eslint '*.{js,mjs}' 'src/**/*.{js,mjs}' 'bin/**/*.{js,mjs}'", "precommit": "lint-staged", - "version": "npm-auto-version", - "start": "node ." + "version": "npm-auto-version" }, "repository": { "type": "git", @@ -50,19 +49,23 @@ "cjs": true }, "dependencies": { - "esm": "^3.0.17" + "axios": "^0.18.0", + "elasticsearch": "^15.2.0", + "esm": "^3.0.17", + "getopts": "^2.2.2", + "node-cron": "^2.0.3" }, "devDependencies": { "@w33ble/npm-auto-tools": "*", "eslint": "^4.9.0", + "eslint-config-airbnb": "^16.1.0", "eslint-config-prettier": "^2.9.0", - "eslint-plugin-prettier": "^2.3.1", "eslint-plugin-import": "^2.7.0", + "eslint-plugin-jsx-a11y": "^6.0.2", + "eslint-plugin-prettier": "^2.3.1", + "eslint-plugin-react": "^7.1.0", "husky": "^0.14.3", "lint-staged": "^7.0.4", - "eslint-config-airbnb": "^16.1.0", - "eslint-plugin-jsx-a11y": "^6.0.2", - "eslint-plugin-react": "^7.1.0", "prettier": "^1.9.0" } } diff --git a/src/index.mjs b/src/index.mjs index 8ace411..a7b2d31 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -1,3 +1,43 @@ -export default function() { - // es6 module code goes here +import elasticsearch from 'elasticsearch'; +import logger from './lib/logger.mjs'; +import { createIndex, bulkInsert } from './lib/data-source.mjs'; +import getData from './lib/get-data.mjs'; + +const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT']; + +export default async function(indexName, opts = {}) { + const client = new elasticsearch.Client({ + host: opts.elasticsearch.host, + log: opts.elasticsearch.log, + }); + + // make sure we can connect to the node + await client.ping(); + + // create the target index + const index = await createIndex(client, indexName); + + const records = await getData(); + logger.debug(`ADS-B records:, ${records.length}`); + + // index all the data + const documents = records.map(rec => ({ + transponder: rec[0], + callsign: rec[1], + origin_country: rec[2], + time_position: rec[3], + last_contact: rec[4], + location: rec[5] && rec[6] ? `${rec[6]},${rec[5]}` : null, + baro_altitude: rec[7], + geo_altitude: rec[13], + on_ground: rec[8], + velocity: rec[9], + vertical_rate: rec[11], + squawk: rec[14], + spi: rec[15], + position_source: positionSourceMap[rec[16]], + })); + + await bulkInsert(client, index, documents); + logger.debug(`Successfully indexed ${records.length} records to ${index}`); } diff --git a/src/lib/data-source.mjs b/src/lib/data-source.mjs new file mode 100644 index 0000000..8fdb4f7 --- /dev/null +++ b/src/lib/data-source.mjs @@ -0,0 +1,80 @@ +import logger from './logger.mjs'; +import { BadRequest } from './es-errors.mjs'; +import zeroPad from './zero-pad.mjs'; + +const doctype = 'doc'; + +// helper for time-based indices +function getIndexName(index) { + const d = new Date(); + const [year, month, day] = [ + d.getFullYear(), + zeroPad(d.getMonth() + 1, 2), + zeroPad(d.getDate(), 2), + ]; + return `${index}-${year}.${month}.${day}`; +} + +export async function createIndex(client, index) { + const realIndex = getIndexName(index); + return client.indices + .create({ + index: realIndex, + body: { + settings: {}, + mappings: { + [doctype]: { + properties: { + transponder: { type: 'keyword' }, + callsign: { type: 'keyword' }, + origin_country: { type: 'keyword' }, + time_position: { type: 'date' }, + last_contact: { type: 'date' }, + location: { type: 'geo_point' }, + baro_altitude: { type: 'float' }, + geo_altitude: { type: 'float' }, + on_ground: { type: 'boolean' }, + velocity: { type: 'float' }, + vertical_rate: { type: 'float' }, + squawk: { type: 'keyword' }, + spi: { type: 'boolean' }, + position_source: { type: 'keyword' }, + }, + }, + }, + }, + }) + .then(res => { + logger.debug(`Index created: ${res.index}`); + return res.index; + }) + .catch(err => { + // check for existing index + if (err instanceof BadRequest) { + logger.debug(`Index exists: ${realIndex}`); + return client.indices.get({ index: realIndex }).then(() => realIndex); + } + + throw err; + }); +} + +export async function createDocument(client, index, body) { + client.index({ + index, + type: doctype, + body, + }); +} + +export async function bulkInsert(client, index, docs) { + const body = docs.reduce((collection, doc) => { + collection.push({ index: { _index: index, _type: doctype } }); + collection.push(doc); + return collection; + }, []); + + client.bulk({ + body, + }); +} diff --git a/src/lib/es-errors.mjs b/src/lib/es-errors.mjs new file mode 100644 index 0000000..d170680 --- /dev/null +++ b/src/lib/es-errors.mjs @@ -0,0 +1,3 @@ +import elasticsearch from 'elasticsearch'; + +export const { BadRequest, NotFound, Forbidden, Conflict } = elasticsearch.errors; diff --git a/src/lib/fetch.mjs b/src/lib/fetch.mjs new file mode 100644 index 0000000..8d37646 --- /dev/null +++ b/src/lib/fetch.mjs @@ -0,0 +1,13 @@ +import axios from 'axios'; + +const api = 'https://opensky-network.org/api'; + +export default axios.create({ + baseURL: api, + timeout: 3000, + responseType: 'json', + headers: { + 'X-Custom-Header': 'foobar', + 'Content-Type': 'application/json; charset=utf-8', + }, +}); diff --git a/src/lib/get-data.mjs b/src/lib/get-data.mjs new file mode 100644 index 0000000..88888d1 --- /dev/null +++ b/src/lib/get-data.mjs @@ -0,0 +1,6 @@ +import fetch from './fetch.mjs'; + +export default async function getData() { + const res = await fetch.get('/states/all'); + return res.data.states; +} diff --git a/src/lib/logger.mjs b/src/lib/logger.mjs new file mode 100644 index 0000000..d4b89fd --- /dev/null +++ b/src/lib/logger.mjs @@ -0,0 +1,35 @@ +/* eslint no-console: 0 */ +import zeroPad from './zero-pad.mjs'; + +const getTimestamp = () => { + const d = new Date(); + return `${[d.getFullYear(), zeroPad(d.getMonth() + 1, 2), zeroPad(d.getDate(), 2)].join('/')} ${[ + zeroPad(d.getHours(), 2), + zeroPad(d.getMinutes(), 2), + zeroPad(d.getSeconds(), 2), + ].join(':')}`; +}; + +const wrapMessage = msg => `[${getTimestamp()}]: ${msg.map(m => JSON.stringify(m)).join(' ')}`; + +const logger = { + log(...args) { + console.log(wrapMessage(args)); + }, + warn(...args) { + console.warn(wrapMessage(args)); + }, + error(...args) { + if (args[0] instanceof Error) { + console.error(getTimestamp(), args[0].stack); + return; + } + console.error(wrapMessage(args)); + }, + debug(...args) { + if (!process.env.DEBUG) return; + console.log(wrapMessage(args)); + }, +}; + +export default logger; diff --git a/src/lib/zero-pad.mjs b/src/lib/zero-pad.mjs new file mode 100644 index 0000000..d110968 --- /dev/null +++ b/src/lib/zero-pad.mjs @@ -0,0 +1,9 @@ +export default function zeroPad(str, len) { + let output = `${str}`; + if (!len || len <= output.length) return str; + + while (output.length < len) { + output = `0${output}`; + } + return output; +} diff --git a/yarn.lock b/yarn.lock index 98a8dfe..dbfeee3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -36,6 +36,13 @@ acorn@^5.5.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.3.tgz#67aa231bf8812974b85235a96771eb6bd07ea279" integrity sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw== +agentkeepalive@^3.4.1: + version "3.5.2" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-3.5.2.tgz#a113924dd3fa24a0bc3b78108c450c2abee00f67" + integrity sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ== + dependencies: + humanize-ms "^1.2.1" + ajv-keywords@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-2.1.1.tgz#617997fc5f60576894c435f940d819e135b80762" @@ -193,6 +200,14 @@ auto-changelog@^1.7.0: parse-github-url "^1.0.1" semver "^5.1.0" +axios@^0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" + integrity sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI= + dependencies: + follow-redirects "^1.3.0" + is-buffer "^1.1.5" + axobject-query@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.0.2.tgz#ea187abe5b9002b377f925d8bf7d1c561adf38f9" @@ -496,6 +511,13 @@ date-fns@^1.27.2: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.29.0.tgz#12e609cdcb935127311d04d33334e2960a2a54e6" integrity sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw== +debug@=3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -582,6 +604,15 @@ doctrine@^2.1.0: dependencies: esutils "^2.0.2" +elasticsearch@^15.2.0: + version "15.2.0" + resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-15.2.0.tgz#234362b5aa743d9c0a925566569ea7813b8f2569" + integrity sha512-jOFcBoEh3Sn3gjUTozInODZTLriJtfppAUC7jnQCUE+OUj8o7GoAyC+L4h/L3ZxmXNFbQCunqVR+nmSofHdo9A== + dependencies: + agentkeepalive "^3.4.1" + chalk "^1.0.0" + lodash "^4.17.10" + elegant-spinner@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/elegant-spinner/-/elegant-spinner-1.0.1.tgz#db043521c95d7e303fd8f345bedc3349cfb0729e" @@ -979,6 +1010,13 @@ flat-cache@^1.2.1: graceful-fs "^4.1.2" write "^0.2.1" +follow-redirects@^1.3.0: + version "1.5.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.9.tgz#c9ed9d748b814a39535716e531b9196a845d89c6" + integrity sha512-Bh65EZI/RU8nx0wbYF9shkFZlqLP+6WT/5FnA3cE/djNSuKNHJEinGGZgu/cQEkeeb2GdFOgenAmn8qaqYke2w== + dependencies: + debug "=3.1.0" + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -1026,7 +1064,7 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= -getopts@^2.0.6: +getopts@^2.0.6, getopts@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.2.tgz#a6ada98961c3d1f099e88f0698fabe716f8a253c" integrity sha512-qX9XmOFViBolzBfWkBKOP/HOO8VSgy6reZMVWo/QvCmwXJLZU5idp4W+L70CM8SPBRBwgrCyU65IKKr40zgizw== @@ -1136,6 +1174,13 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047" integrity sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w== +humanize-ms@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" + integrity sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0= + dependencies: + ms "^2.0.0" + husky@^0.14.3: version "0.14.3" resolved "https://registry.yarnpkg.com/husky/-/husky-0.14.3.tgz#c69ed74e2d2779769a17ba8399b54ce0b63c12c3" @@ -1744,7 +1789,7 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@^2.1.1: +ms@^2.0.0, ms@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== @@ -1781,6 +1826,14 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-cron@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-2.0.3.tgz#b9649784d0d6c00758410eef22fa54a10e3f602d" + integrity sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg== + dependencies: + opencollective-postinstall "^2.0.0" + tz-offset "0.0.1" + node-fetch@^2.1.2, node-fetch@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5" @@ -1881,6 +1934,11 @@ onetime@^2.0.0: dependencies: mimic-fn "^1.0.0" +opencollective-postinstall@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz#798e83e168f7b91949061c2683f762af747f17cc" + integrity sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ== + optimist@^0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686" @@ -2571,6 +2629,11 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= +tz-offset@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/tz-offset/-/tz-offset-0.0.1.tgz#fef920257024d3583ed9072a767721a18bdb8a76" + integrity sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ== + uglify-js@^3.1.4: version "3.4.9" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.9.tgz#af02f180c1207d76432e473ed24a28f4a782bae3"