fix: correctly formatted ads-b exchange data

CHANGELOG.md

@@ -1,20 +1,5 @@
 ### Changelog
 
-#### [v2.1.1](https://git.w33ble.com/w33ble/adsb-index/compare/v2.1.0...v2.1.1) (6 November 2018)
-- fix: saner defaults [`c5e8b7f`](https://git.w33ble.com/w33ble/adsb-index/commit/c5e8b7f178a8bbc76b72eea08afc4d578baababb)
-
-#### [v2.1.0](https://git.w33ble.com/w33ble/adsb-index/compare/v2.0.1...v2.1.0) (2 November 2018)
-- feat: add raw version of text fields [`6591e7e`](https://git.w33ble.com/w33ble/adsb-index/commit/6591e7eaa8c9c317a55632f856310cdd3f783464)
-
-#### [v2.0.1](https://git.w33ble.com/w33ble/adsb-index/compare/v2.0.0...v2.0.1) (2 November 2018)
-- fix: format undefined fields [`024a43e`](https://git.w33ble.com/w33ble/adsb-index/commit/024a43e20b7ebcf79ef5625df2021a41fa9b2ce9)
-- fix: exit on errors [`f7523ae`](https://git.w33ble.com/w33ble/adsb-index/commit/f7523ae5e20414dfec597050051566adbba9a310)
-
-### [v2.0.0](https://git.w33ble.com/w33ble/adsb-index/compare/v1.0.1...v2.0.0) (2 November 2018)
-- feat: add ads-b exchange source [`9f0da54`](https://git.w33ble.com/w33ble/adsb-index/commit/9f0da54ccfc639ec6367ea85ec39bea117fd3108)
-- feat: use ads-b exchange data [`89261d0`](https://git.w33ble.com/w33ble/adsb-index/commit/89261d0010315ad248dcfdcf25916d952de648d8)
-- fix: correctly formatted ads-b exchange data [`d290165`](https://git.w33ble.com/w33ble/adsb-index/commit/d2901658990a5984ef69fc34f7372e3894a3ac74)
-
 #### [v1.0.1](https://git.w33ble.com/w33ble/adsb-index/compare/v1.0.0...v1.0.1) (2 November 2018)
 - fix: format api data correctly [`6def944`](https://git.w33ble.com/w33ble/adsb-index/commit/6def9449be415c50390a2eaa4fd124bd5fa548b6)
 

bin/index.js

@@ -1,7 +1,7 @@
 /* eslint no-global-assign: 0 no-console: 0 */
 require = require('esm')(module);
 const getopts = require('getopts');
-const runInterval = require('interval-promise');
+const cron = require('node-cron');
 const mod = require('../src/index.mjs').default;
 const logger = require('../src/lib/logger.mjs').default;
 
@@ -21,7 +21,7 @@ const { index, interval, lat, lon, radius, ...elasticsearch } = getopts(process.
     interval: 0,
     lat: 33.433638,
     lon: -112.008113,
-    radius: 200,
+    radius: 1000,
   },
 });
 
@@ -31,7 +31,6 @@ function handleError(err) {
 }
 
 async function fetchAndIndex() {
-  logger.debug('Fetching and indexing data...');
   return mod(index, { elasticsearch, filter: { lat, lon, radius } });
 }
 
@@ -39,10 +38,14 @@ async function run() {
   // initial kickoff
   await fetchAndIndex().catch(handleError);
 
-  // scheduled running
   if (interval === 0) return;
-  logger.debug(`Starting interval (${interval}s)...`);
-  runInterval(() => fetchAndIndex().catch(handleError), interval * 1000);
+
+  // scheduled running
+  logger.debug(`Starting cron (${interval}s)...`);
+  cron.schedule(`${interval} * * * * *`, () => {
+    logger.debug('Running cron...');
+    fetchAndIndex();
+  });
 }
 
 run().catch(handleError);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "adsb-index",
-  "version": "2.1.1",
+  "version": "1.0.1",
   "private": true,
   "description": "ADS-B indexing script",
   "bin": "bin/index.js",
@@ -53,7 +53,7 @@
     "elasticsearch": "^15.2.0",
     "esm": "^3.0.17",
     "getopts": "^2.2.2",
-    "interval-promise": "^1.3.0"
+    "node-cron": "^2.0.3"
   },
   "devDependencies": {
     "@w33ble/npm-auto-tools": "*",

src/lib/data-source.mjs

@@ -26,8 +26,8 @@ export async function createIndex(client, index) {
         mappings: {
           [doctype]: {
             properties: {
-              operator: { type: 'text', fields: { raw: { type: 'keyword' } } },
-              aircraft: { type: 'text', fields: { raw: { type: 'keyword' } } },
+              operator: { type: 'keyword' },
+              aircraft: { type: 'text' },
               aircraft_manufacturer: { type: 'keyword' },
               transponder: { type: 'keyword' },
               callsign: { type: 'keyword' },
@@ -45,8 +45,8 @@ export async function createIndex(client, index) {
               squawk: { type: 'keyword' },
               spi: { type: 'boolean' },
               position_source: { type: 'keyword' },
-              from: { type: 'text', fields: { raw: { type: 'keyword' } } },
-              to: { type: 'text', fields: { raw: { type: 'keyword' } } },
+              from: { type: 'text' },
+              to: { type: 'text' },
             },
           },
         },

src/lib/fetch.mjs

@@ -1,7 +1,7 @@
 import axios from 'axios';
 
 export default axios.create({
-  timeout: 3000,
+  timeout: 5000,
   responseType: 'json',
   headers: {
     'Content-Type': 'application/json; charset=utf-8',

src/lib/get-data.mjs

@@ -1,12 +1,5 @@
 import fetch from './fetch.mjs';
 
-const formatNumber = str => str && Number(`${str}`.replace(/[^0-9.-]/, ''));
-const formatString = str => {
-  if (str == null) return 'N/A';
-  if (typeof str !== 'string') return str;
-  return str && str.length && str !== 'undefined' ? str.trim() : 'N/A';
-};
-
 export async function getOpenskyData() {
   const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT'];
   const res = await fetch.get(`https://opensky-network.org/api/states/all`);
@@ -32,32 +25,33 @@ export async function getOpenskyData() {
 
 export async function getAdbsExchangeData({ lat, lon, radius }) {
   const positionSourceMap = ['Unknown', 'Mode-S', 'ADS-B', 'ADS-B', 'ADS-B', 'ADS-B'];
-
   const res = await fetch.get(
     `http://public-api.adsbexchange.com/VirtualRadar/AircraftList.json?lat=${lat}&lng=${lon}&fDstL=0&fDstU=${radius}`
   );
 
+  const invalidNumber = /[^0-9.-]/;
+
   return res.data.acList.map(rec => ({
-    transponder: formatString(`${rec.Icao}`.toLowerCase()),
-    callsign: formatString(rec.Call),
+    transponder: `${rec.Icao}`.toLowerCase(),
+    callsign: `${rec.Call}`.trim(),
     origin_country: rec.Cou,
     time_position: new Date(rec.PosTime),
     last_contact: new Date(rec.PosTime),
     location: rec.Lat && rec.Long ? `${rec.Lat},${rec.Long}` : null,
     lat: rec.Lat,
     lon: rec.Long,
-    baro_altitude: formatNumber(rec.Alt),
-    geo_altitude: formatNumber(rec.GAlt),
+    baro_altitude: rec.Alt.replace(invalidNumber, ''),
+    geo_altitude: rec.Galt.replace(invalidNumber, ''),
     on_ground: rec.Gnd,
-    velocity: formatNumber(rec.Spd),
-    vertical_rate: formatNumber(rec.Vsi),
+    velocity: rec.Spd.replace(invalidNumber, ''),
+    vertical_rate: rec.Vsi.replace(invalidNumber, ''),
     squawk: rec.Sqk,
     spi: false,
     position_source: positionSourceMap[rec.Trt],
-    operator: formatString(rec.Op),
-    aircraft: formatString(rec.Mdl),
-    aircraft_manufacturer: formatString(rec.Man),
-    from: formatString(rec.From),
-    to: formatString(rec.To),
+    operator: rec.Op,
+    aircraft: rec.Mdl,
+    aircraft_manufacturer: rec.Man,
+    from: rec.From,
+    to: rec.To,
   }));
 }

yarn.lock

@@ -1245,11 +1245,6 @@ inquirer@^3.0.6:
     strip-ansi "^4.0.0"
     through "^2.3.6"
 
-interval-promise@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/interval-promise/-/interval-promise-1.3.0.tgz#26fc4fbcd18605162d0dd012c2cc5657c4f8bc22"
-  integrity sha512-AebPZJ0o6D1jWg487njjrhfzBp6/hfmtWCMQhXr0SXD7Zg5RexQe4Y6joYiGieJziP7W7WEMQWxFKSbywXmhbw==
-
 is-accessor-descriptor@^0.1.6:
   version "0.1.6"
   resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6"
@@ -1831,6 +1826,14 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
+node-cron@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-2.0.3.tgz#b9649784d0d6c00758410eef22fa54a10e3f602d"
+  integrity sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==
+  dependencies:
+    opencollective-postinstall "^2.0.0"
+    tz-offset "0.0.1"
+
 node-fetch@^2.1.2, node-fetch@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
@@ -1931,6 +1934,11 @@ onetime@^2.0.0:
   dependencies:
     mimic-fn "^1.0.0"
 
+opencollective-postinstall@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.1.tgz#798e83e168f7b91949061c2683f762af747f17cc"
+  integrity sha512-saQQ9hjLwu/oS0492eyYotoh+bra1819cfAT5rjY/e4REWwuc8IgZ844Oo44SiftWcJuBiqp0SA0BFVbmLX0IQ==
+
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
@@ -2621,6 +2629,11 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
+tz-offset@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/tz-offset/-/tz-offset-0.0.1.tgz#fef920257024d3583ed9072a767721a18bdb8a76"
+  integrity sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==
+
 uglify-js@^3.1.4:
   version "3.4.9"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.4.9.tgz#af02f180c1207d76432e473ed24a28f4a782bae3"