2.0.0

export a function for the specific opensky data, re-org code

CHANGELOG.md

@@ -1,6 +1,14 @@
 ### Changelog
 
-#### 1.0.0 (2 November 2018)
+### [v2.0.0](https://git.w33ble.com/w33ble/adsb-index/compare/v1.0.1...v2.0.0) (2 November 2018)
+- feat: add ads-b exchange source [`9f0da54`](https://git.w33ble.com/w33ble/adsb-index/commit/9f0da54ccfc639ec6367ea85ec39bea117fd3108)
+- feat: use ads-b exchange data [`89261d0`](https://git.w33ble.com/w33ble/adsb-index/commit/89261d0010315ad248dcfdcf25916d952de648d8)
+- fix: correctly formatted ads-b exchange data [`d290165`](https://git.w33ble.com/w33ble/adsb-index/commit/d2901658990a5984ef69fc34f7372e3894a3ac74)
+
+#### [v1.0.1](https://git.w33ble.com/w33ble/adsb-index/compare/v1.0.0...v1.0.1) (2 November 2018)
+- fix: format api data correctly [`6def944`](https://git.w33ble.com/w33ble/adsb-index/commit/6def9449be415c50390a2eaa4fd124bd5fa548b6)
+
+#### v1.0.0 (2 November 2018)
 - feat: index location as raw numbers [`98e54c8`](https://git.w33ble.com/w33ble/adsb-index/commit/98e54c89d99d01f9dd7762e2851449740e0c8571)
 - fix: save timestamps correctly [`20bb946`](https://git.w33ble.com/w33ble/adsb-index/commit/20bb9460d17af65950bb5beffab3a3fc6a075a9a)
 - feat: add basic auth support [`420c845`](https://git.w33ble.com/w33ble/adsb-index/commit/420c845cbc70a7a816ea92986c4e10c206194dca)

bin/index.js

@@ -1,11 +1,11 @@
 /* eslint no-global-assign: 0 no-console: 0 */
 require = require('esm')(module);
 const getopts = require('getopts');
-const cron = require('node-cron');
+const runInterval = require('interval-promise');
 const mod = require('../src/index.mjs').default;
 const logger = require('../src/lib/logger.mjs').default;
 
-const { index, interval, ...elasticsearch } = getopts(process.argv.slice(2), {
+const { index, interval, lat, lon, radius, ...elasticsearch } = getopts(process.argv.slice(2), {
   string: ['host', 'auth', 'log', 'index'],
   alias: {
     h: 'host',
@@ -19,6 +19,9 @@ const { index, interval, ...elasticsearch } = getopts(process.argv.slice(2), {
     log: 'error',
     index: 'adsb-data',
     interval: 0,
+    lat: 33.433638,
+    lon: -112.008113,
+    radius: 1000,
   },
 });
 
@@ -28,21 +31,18 @@ function handleError(err) {
 }
 
 async function fetchAndIndex() {
-  return mod(index, { elasticsearch });
+  logger.debug('Fetching and indexing data...');
+  return mod(index, { elasticsearch, filter: { lat, lon, radius } });
 }
 
 async function run() {
   // initial kickoff
   await fetchAndIndex().catch(handleError);
 
-  if (interval === 0) return;
-
   // scheduled running
-  logger.debug(`Starting cron (${interval}s)...`);
-  cron.schedule(`${interval} * * * * *`, () => {
-    logger.debug('Running cron...');
-    fetchAndIndex();
-  });
+  if (interval === 0) return;
+  logger.debug(`Starting interval (${interval}s)...`);
+  runInterval(fetchAndIndex, interval * 1000);
 }
 
 run().catch(handleError);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "adsb-index",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "private": true,
   "description": "ADS-B indexing script",
   "bin": "bin/index.js",
@@ -53,7 +53,7 @@
     "elasticsearch": "^15.2.0",
     "esm": "^3.0.17",
     "getopts": "^2.2.2",
-    "node-cron": "^2.0.3"
+    "interval-promise": "^1.3.0"
   },
   "devDependencies": {
     "@w33ble/npm-auto-tools": "*",

src/index.mjs

@@ -1,9 +1,7 @@
 import elasticsearch from 'elasticsearch';
 import logger from './lib/logger.mjs';
 import { createIndex, bulkInsert } from './lib/data-source.mjs';
-import getData from './lib/get-data.mjs';
-
-const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT'];
+import { getAdbsExchangeData } from './lib/get-data.mjs';
 
 export default async function(indexName, opts = {}) {
   const client = new elasticsearch.Client({
@@ -18,29 +16,9 @@ export default async function(indexName, opts = {}) {
   // create the target index
   const index = await createIndex(client, indexName);
 
-  const records = await getData();
-  logger.debug(`ADS-B records:, ${records.length}`);
+  const records = await getAdbsExchangeData(opts.filter);
+  logger.debug(`Record count:, ${records.length}`);
 
-  // index all the data
-  const documents = records.map(rec => ({
-    transponder: rec[0],
-    callsign: rec[1],
-    origin_country: rec[2],
-    time_position: new Date(rec[3] * 1000),
-    last_contact: new Date(rec[4] * 1000),
-    location: rec[5] && rec[6] ? `${rec[6]},${rec[5]}` : null,
-    lat: rec[6],
-    lon: rec[5],
-    baro_altitude: rec[7],
-    geo_altitude: rec[13],
-    on_ground: rec[8],
-    velocity: rec[9],
-    vertical_rate: rec[11],
-    squawk: rec[14],
-    spi: rec[15],
-    position_source: positionSourceMap[rec[16]],
-  }));
-
-  await bulkInsert(client, index, documents);
+  await bulkInsert(client, index, records);
   logger.debug(`Successfully indexed ${records.length} records to ${index}`);
 }

src/lib/data-source.mjs

@@ -26,6 +26,9 @@ export async function createIndex(client, index) {
         mappings: {
           [doctype]: {
             properties: {
+              operator: { type: 'keyword' },
+              aircraft: { type: 'text' },
+              aircraft_manufacturer: { type: 'keyword' },
               transponder: { type: 'keyword' },
               callsign: { type: 'keyword' },
               origin_country: { type: 'keyword' },
@@ -42,6 +45,8 @@ export async function createIndex(client, index) {
               squawk: { type: 'keyword' },
               spi: { type: 'boolean' },
               position_source: { type: 'keyword' },
+              from: { type: 'text' },
+              to: { type: 'text' },
             },
           },
         },

src/lib/fetch.mjs

@@ -1,13 +1,9 @@
 import axios from 'axios';
 
-const api = 'https://opensky-network.org/api';
-
 export default axios.create({
-  baseURL: api,
-  timeout: 3000,
+  timeout: 5000,
   responseType: 'json',
   headers: {
-    'X-Custom-Header': 'foobar',
     'Content-Type': 'application/json; charset=utf-8',
   },
 });

src/lib/get-data.mjs

@@ -1,6 +1,57 @@
 import fetch from './fetch.mjs';
 
-export default async function getData() {
-  const res = await fetch.get('/states/all');
-  return res.data.states;
+export async function getOpenskyData() {
+  const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT'];
+  const res = await fetch.get(`https://opensky-network.org/api/states/all`);
+  return res.data.states.map(rec => ({
+    transponder: `${rec[0]}`.toLowerCase(),
+    callsign: `${rec[1]}`.trim(),
+    origin_country: rec[2],
+    time_position: new Date(rec[3] * 1000),
+    last_contact: new Date(rec[4] * 1000),
+    location: rec[5] && rec[6] ? `${rec[6]},${rec[5]}` : null,
+    lat: rec[6],
+    lon: rec[5],
+    baro_altitude: rec[7],
+    geo_altitude: rec[13],
+    on_ground: rec[8],
+    velocity: rec[9],
+    vertical_rate: rec[11],
+    squawk: rec[14],
+    spi: rec[15],
+    position_source: positionSourceMap[rec[16]],
+  }));
+}
+
+export async function getAdbsExchangeData({ lat, lon, radius }) {
+  const positionSourceMap = ['Unknown', 'Mode-S', 'ADS-B', 'ADS-B', 'ADS-B', 'ADS-B'];
+  const formatNumber = str => str && Number(`${str}`.replace(/[^0-9.-]/, ''));
+
+  const res = await fetch.get(
+    `http://public-api.adsbexchange.com/VirtualRadar/AircraftList.json?lat=${lat}&lng=${lon}&fDstL=0&fDstU=${radius}`
+  );
+
+  return res.data.acList.map(rec => ({
+    transponder: `${rec.Icao}`.toLowerCase(),
+    callsign: `${rec.Call}`.trim(),
+    origin_country: rec.Cou,
+    time_position: new Date(rec.PosTime),
+    last_contact: new Date(rec.PosTime),
+    location: rec.Lat && rec.Long ? `${rec.Lat},${rec.Long}` : null,
+    lat: rec.Lat,
+    lon: rec.Long,
+    baro_altitude: formatNumber(rec.Alt),
+    geo_altitude: formatNumber(rec.GAlt),
+    on_ground: rec.Gnd,
+    velocity: formatNumber(rec.Spd),
+    vertical_rate: formatNumber(rec.Vsi),
+    squawk: rec.Sqk,
+    spi: false,
+    position_source: positionSourceMap[rec.Trt],
+    operator: rec.Op,
+    aircraft: rec.Mdl,
+    aircraft_manufacturer: rec.Man,
+    from: rec.From,
+    to: rec.To,
+  }));
 }

yarn.lock

@@ -1245,6 +1245,11 @@ inquirer@^3.0.6:
     strip-ansi "^4.0.0"
     through "^2.3.6"
 
+interval-promise@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/interval-promise/-/interval-promise-1.3.0.tgz#26fc4fbcd18605162d0dd012c2cc5657c4f8bc22"
+  integrity sha512-AebPZJ0o6D1jWg487njjrhfzBp6/hfmtWCMQhXr0SXD7Zg5RexQe4Y6joYiGieJziP7W7WEMQWxFKSbywXmhbw==
+
 is-accessor-descriptor@^0.1.6:
   version "0.1.6"
   resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6"
@@ -1826,14 +1831,6 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
-node-cron@^2.0.3:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-2.0.3.tgz#b9649784d0d6c00758410eef22fa54a10e3f602d"
-  integrity sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==
-  dependencies:
-    opencollective-postinstall "^2.0.0"
-    tz-offset "0.0.1"
-
 node-fetch@^2.1.2, node-fetch@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
@@ -1934,11 +1931,6 @@ onetime@^2.0.0:
   dependencies:
     mimic-fn "^1.0.0"
 
-opencollective-postinstall@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz#798e83e168f7b91949061c2683f762af747f17cc"
-  integrity sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==
-
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
@@ -2629,11 +2621,6 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
-tz-offset@0.0.1:
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/tz-offset/-/tz-offset-0.0.1.tgz#fef920257024d3583ed9072a767721a18bdb8a76"
-  integrity sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==
-
 uglify-js@^3.1.4:
   version "3.4.9"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.9.tgz#af02f180c1207d76432e473ed24a28f4a782bae3"