w33ble/strain-tools
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

feat: add tags to scraper output

Browse Source
This commit is contained in:
joe fleming
2018-08-31 10:39:48 -07:00
parent 22a95c3fe8
commit 804fac1afb
2 changed files with 163 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
packages/scraper/src/leafly.mjs
View File

@@ -12,6 +12,8 @@ const xhr = axios.create({
}, },
}); });
const pSeries = tasks => tasks.reduce((c, task) => c.then(task), Promise.resolve());
const getPage = async num => { const getPage = async num => {
const url = `https://www.leafly.com/explore/page-${num}`; const url = `https://www.leafly.com/explore/page-${num}`;
const response = await xhr.get(url, { const response = await xhr.get(url, {
@@ -45,13 +47,25 @@ export default async function scrapeLeafly(startFrom = 1, endAt = Infinity) {
let finished = false; let finished = false;
const db = await low(adapter); const db = await low(adapter);
await db.defaults({ strains: [] }).write(); async function writeTag(type, tag) {
const res = await db
.get(type)
.indexOf(tag)
.value();
while (!finished) { if (res < 0) {
console.log(`Fetching page ${pageNum}`); await db
const data = await getPage(pageNum); .get(type)
.push(tag)
.write();
}
}
data.strains.forEach(async strain => { async function writeTags(type, tags) {
await pSeries(tags.map(tag => () => writeTag(type, tag)));
}
async function writeDoc(strain) {
// check for value // check for value
const doc = db const doc = db
.get('strains') .get('strains')
@@ -65,8 +79,31 @@ export default async function scrapeLeafly(startFrom = 1, endAt = Infinity) {
.get('strains') .get('strains')
.push(strain) .push(strain)
.write(); .write();
await writeTags('effects', strain.effects);
await writeTags('negative_effects', strain.negative_effects);
await writeTags('uses', strain.uses);
await writeTags('conditions', strain.conditions);
await writeTags('flavors', strain.flavors);
} }
}); }
await db
.defaults({
strains: [],
effects: [],
negative_effects: [],
uses: [],
conditions: [],
flavors: [],
})
.write();
while (!finished) {
console.log(`Fetching page ${pageNum}`);
const data = await getPage(pageNum);
await pSeries(data.strains.map(strain => () => writeDoc(strain)));
if (pageNum >= endAt || !data.strains.length || data.page.isLastPage) finished = true; if (pageNum >= endAt || !data.strains.length || data.page.isLastPage) finished = true;
pageNum += 1; pageNum += 1;

65
packages/search-site/src/index.ejs
View File

@@ -10,7 +10,9 @@
<body> <body>
<noscript><h1>You're going to want to enable JavaScript</h1></noscript> <noscript><h1>You're going to want to enable JavaScript</h1></noscript>
<section class="section"> <section class="section">
<form id="search-form" @submit.prevent="handleSubmit">
<div class="container"> <div class="container">
<h1 class="title"> <h1 class="title">
Strain Search Strain Search
@@ -20,7 +22,7 @@
<div class="field"> <div class="field">
<label class="label">Search By Name</label> <label class="label">Search By Name</label>
<div class="control"> <div class="control">
<input class="input" type="text" placeholder="Text input"> <input ref="name" class="input" type="text" placeholder="Text input">
</div> </div>
</div> </div>
@@ -32,7 +34,7 @@
<div class="field"> <div class="field">
<label class="label">Desired Effects</label> <label class="label">Desired Effects</label>
<div class="select is-multiple"> <div class="select is-multiple">
<select multiple size="6"> <select ref="effects" multiple size="6">
<option value="Happy">Happy</option> <option value="Happy">Happy</option>
<option value="Euphoric">Euphoric</option> <option value="Euphoric">Euphoric</option>
<option value="Sleepy">Sleepy</option> <option value="Sleepy">Sleepy</option>
@@ -44,7 +46,7 @@
<div class="field"> <div class="field">
<label class="label">Medical Use</label> <label class="label">Medical Use</label>
<div class="select is-multiple"> <div class="select is-multiple">
<select multiple size="6"> <select ref="uses" multiple size="6">
<option value="Pain">Pain</option> <option value="Pain">Pain</option>
<option value="Depression">Depression</option> <option value="Depression">Depression</option>
<option value="Insomnia">Insomnia</option> <option value="Insomnia">Insomnia</option>
@@ -61,7 +63,7 @@
<div class="field"> <div class="field">
<label class="label">Condition</label> <label class="label">Condition</label>
<div class="select is-multiple"> <div class="select is-multiple">
<select multiple size="6"> <select ref="conditions" multiple size="6">
<option value="PTSD">PTSD</option> <option value="PTSD">PTSD</option>
</select> </select>
</div> </div>
@@ -72,7 +74,7 @@
<div class="field"> <div class="field">
<label class="label">Flavor</label> <label class="label">Flavor</label>
<div class="select is-multiple"> <div class="select is-multiple">
<select multiple size="6"> <select ref="flavors" multiple size="6">
<option value="Sweet">Sweet</option> <option value="Sweet">Sweet</option>
<option value="Woody">Woody</option> <option value="Woody">Woody</option>
</select> </select>
@@ -86,28 +88,59 @@
<!-- Form Submits --> <!-- Form Submits -->
<div class="field is-grouped"> <div class="field is-grouped">
<div class="control"> <div class="control">
<button class="button is-link">Submit</button> <button type="submit" class="button is-link">Submit</button>
</div> </div>
<div class="control"> <div class="control">
<button class="button is-text">Clear</button> <button type="button" class="button is-text" @click.prevent="resetForm">Clear</button>
</div> </div>
</div> </div>
</div> </div>
</form>
</section> </section>
<section class="section"> <section class="section">
<div class="container" id="strain"></div> <div class="container" id="strain"></div>
</section> </section>
<script src="https://unpkg.com/lunr@2.3.1/lunr.js"></script>
<script>
(function () {
const $ = s => Array.from(document.querySelectorAll(s));
const $$ = s => [document.querySelector(s)];
const attr = node => name => node && node.getAttribute(name);
var data = <%- data %>; <script src="https://unpkg.com/lunr@2.3.1/lunr.js"></script>
<script src="https://unpkg.com/vue@2.5.17/dist/vue.js"></script>
<script src="https://unpkg.com/mitt@1.1.3/dist/mitt.umd.js"></script>
<script>
// lunr = window.lunr
// mitt = window.mitt
(function ({ mitt, lunr }) {
const data = <%- data %>;
const emitter = mitt();
// debugging...
const target = document.querySelector('#strain').textContent = data.strains[20].name; const target = document.querySelector('#strain').textContent = data.strains[20].name;
})();
// helpers
const getMultiValues = node => Array.from(node.selectedOptions).map(o => o.value);
// form handler
new Vue({
el: '#search-form',
data() {
return {};
},
methods: {
handleSubmit() {
const requirements = {
name: this.$refs.name.value,
effects: getMultiValues(this.$refs.effects),
uses: getMultiValues(this.$refs.uses),
conditions: getMultiValues(this.$refs.conditions),
flavors: getMultiValues(this.$refs.flavors),
}
console.log(requirements)
},
resetForm() {
this.$el.reset();
}
}
})
})(this);
</script> </script>
</body> </body>
</html> </html>