feat: working data download and indexing

This commit is contained in:
2018-11-01 16:57:10 -07:00
parent 2067fd63ca
commit 571e6dfd75
12 changed files with 289 additions and 22 deletions

View File

@@ -1,3 +1,43 @@
export default function() {
// es6 module code goes here
import elasticsearch from 'elasticsearch';
import logger from './lib/logger.mjs';
import { createIndex, bulkInsert } from './lib/data-source.mjs';
import getData from './lib/get-data.mjs';
const positionSourceMap = ['ADS-B', 'ASTERIX', 'MLAT'];
export default async function(indexName, opts = {}) {
const client = new elasticsearch.Client({
host: opts.elasticsearch.host,
log: opts.elasticsearch.log,
});
// make sure we can connect to the node
await client.ping();
// create the target index
const index = await createIndex(client, indexName);
const records = await getData();
logger.debug(`ADS-B records:, ${records.length}`);
// index all the data
const documents = records.map(rec => ({
transponder: rec[0],
callsign: rec[1],
origin_country: rec[2],
time_position: rec[3],
last_contact: rec[4],
location: rec[5] && rec[6] ? `${rec[6]},${rec[5]}` : null,
baro_altitude: rec[7],
geo_altitude: rec[13],
on_ground: rec[8],
velocity: rec[9],
vertical_rate: rec[11],
squawk: rec[14],
spi: rec[15],
position_source: positionSourceMap[rec[16]],
}));
await bulkInsert(client, index, documents);
logger.debug(`Successfully indexed ${records.length} records to ${index}`);
}