w33ble/strain-tools
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

feat: add tags to scraper output

Browse Source
This commit is contained in:
joe fleming
2018-08-31 10:39:48 -07:00
parent 22a95c3fe8
commit 804fac1afb
2 changed files with 163 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
packages/scraper/src/leafly.mjs
View File

@@ -12,6 +12,8 @@ const xhr = axios.create({
},
});
const pSeries = tasks => tasks.reduce((c, task) => c.then(task), Promise.resolve());
const getPage = async num => {
const url = `https://www.leafly.com/explore/page-${num}`;
const response = await xhr.get(url, {
@@ -45,28 +47,63 @@ export default async function scrapeLeafly(startFrom = 1, endAt = Infinity) {
let finished = false;
const db = await low(adapter);
await db.defaults({ strains: [] }).write();
async function writeTag(type, tag) {
const res = await db
.get(type)
.indexOf(tag)
.value();
if (res < 0) {
await db
.get(type)
.push(tag)
.write();
}
}
async function writeTags(type, tags) {
await pSeries(tags.map(tag => () => writeTag(type, tag)));
}
async function writeDoc(strain) {
// check for value
const doc = db
.get('strains')
.filter({ id: strain.id })
.first()
.value();
if (!doc) {
console.log(`Adding ${strain.id}, ${strain.name}`);
await db
.get('strains')
.push(strain)
.write();
await writeTags('effects', strain.effects);
await writeTags('negative_effects', strain.negative_effects);
await writeTags('uses', strain.uses);
await writeTags('conditions', strain.conditions);
await writeTags('flavors', strain.flavors);
}
}
await db
.defaults({
strains: [],
effects: [],
negative_effects: [],
uses: [],
conditions: [],
flavors: [],
})
.write();
while (!finished) {
console.log(`Fetching page ${pageNum}`);
const data = await getPage(pageNum);
data.strains.forEach(async strain => {
// check for value
const doc = db
.get('strains')
.filter({ id: strain.id })
.first()
.value();
if (!doc) {
console.log(`Adding ${strain.id}, ${strain.name}`);
await db
.get('strains')
.push(strain)
.write();
}
});
await pSeries(data.strains.map(strain => () => writeDoc(strain)));
if (pageNum >= endAt || !data.strains.length || data.page.isLastPage) finished = true;
pageNum += 1;

185
packages/search-site/src/index.ejs
View File

@@ -10,104 +10,137 @@
<body>
<noscript><h1>You're going to want to enable JavaScript</h1></noscript>
<section class="section">
<div class="container">
<h1 class="title">
Strain Search
</h1>
<form id="search-form" @submit.prevent="handleSubmit">
<div class="container">
<h1 class="title">
Strain Search
</h1>
<!-- Name Search Input -->
<div class="field">
<label class="label">Search By Name</label>
<div class="control">
<input class="input" type="text" placeholder="Text input">
<!-- Name Search Input -->
<div class="field">
<label class="label">Search By Name</label>
<div class="control">
<input ref="name" class="input" type="text" placeholder="Text input">
</div>
</div>
</div>
<!-- Multi-Selects -->
<div class="columns">
<div class="column">
<div class="columns is-mobile">
<div class="column">
<div class="field">
<label class="label">Desired Effects</label>
<div class="select is-multiple">
<select multiple size="6">
<option value="Happy">Happy</option>
<option value="Euphoric">Euphoric</option>
<option value="Sleepy">Sleepy</option>
</select>
<!-- Multi-Selects -->
<div class="columns">
<div class="column">
<div class="columns is-mobile">
<div class="column">
<div class="field">
<label class="label">Desired Effects</label>
<div class="select is-multiple">
<select ref="effects" multiple size="6">
<option value="Happy">Happy</option>
<option value="Euphoric">Euphoric</option>
<option value="Sleepy">Sleepy</option>
</select>
</div>
</div>
</div>
<div class="column">
<div class="field">
<label class="label">Medical Use</label>
<div class="select is-multiple">
<select ref="uses" multiple size="6">
<option value="Pain">Pain</option>
<option value="Depression">Depression</option>
<option value="Insomnia">Insomnia</option>
</select>
</div>
</div>
</div>
</div>
<div class="column">
<div class="field">
<label class="label">Medical Use</label>
<div class="select is-multiple">
<select multiple size="6">
<option value="Pain">Pain</option>
<option value="Depression">Depression</option>
<option value="Insomnia">Insomnia</option>
</select>
</div>
<div class="column">
<div class="columns is-mobile">
<div class="column">
<div class="field">
<label class="label">Condition</label>
<div class="select is-multiple">
<select ref="conditions" multiple size="6">
<option value="PTSD">PTSD</option>
</select>
</div>
</div>
</div>
<div class="column">
<div class="field">
<label class="label">Flavor</label>
<div class="select is-multiple">
<select ref="flavors" multiple size="6">
<option value="Sweet">Sweet</option>
<option value="Woody">Woody</option>
</select>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="column">
<div class="columns is-mobile">
<div class="column">
<div class="field">
<label class="label">Condition</label>
<div class="select is-multiple">
<select multiple size="6">
<option value="PTSD">PTSD</option>
</select>
</div>
</div>
</div>
<div class="column">
<div class="field">
<label class="label">Flavor</label>
<div class="select is-multiple">
<select multiple size="6">
<option value="Sweet">Sweet</option>
<option value="Woody">Woody</option>
</select>
</div>
</div>
</div>
<!-- Form Submits -->
<div class="field is-grouped">
<div class="control">
<button type="submit" class="button is-link">Submit</button>
</div>
<div class="control">
<button type="button" class="button is-text" @click.prevent="resetForm">Clear</button>
</div>
</div>
</div>
<!-- Form Submits -->
<div class="field is-grouped">
<div class="control">
<button class="button is-link">Submit</button>
</div>
<div class="control">
<button class="button is-text">Clear</button>
</div>
</div>
</div>
</form>
</section>
<section class="section">
<div class="container" id="strain"></div>
</section>
<script src="https://unpkg.com/lunr@2.3.1/lunr.js"></script>
<script>
(function () {
const $ = s => Array.from(document.querySelectorAll(s));
const $$ = s => [document.querySelector(s)];
const attr = node => name => node && node.getAttribute(name);
var data = <%- data %>;
<script src="https://unpkg.com/lunr@2.3.1/lunr.js"></script>
<script src="https://unpkg.com/vue@2.5.17/dist/vue.js"></script>
<script src="https://unpkg.com/mitt@1.1.3/dist/mitt.umd.js"></script>
<script>
// lunr = window.lunr
// mitt = window.mitt
(function ({ mitt, lunr }) {
const data = <%- data %>;
const emitter = mitt();
// debugging...
const target = document.querySelector('#strain').textContent = data.strains[20].name;
})();
// helpers
const getMultiValues = node => Array.from(node.selectedOptions).map(o => o.value);
// form handler
new Vue({
el: '#search-form',
data() {
return {};
},
methods: {
handleSubmit() {
const requirements = {
name: this.$refs.name.value,
effects: getMultiValues(this.$refs.effects),
uses: getMultiValues(this.$refs.uses),
conditions: getMultiValues(this.$refs.conditions),
flavors: getMultiValues(this.$refs.flavors),
}
console.log(requirements)
},
resetForm() {
this.$el.reset();
}
}
})
})(this);
</script>
</body>
</html>