added eslint, some cleanup and fetch in english

This commit is contained in:
2020-05-24 21:28:05 +03:00
parent 8c5fba8788
commit 1b22ac4ceb
8 changed files with 1312 additions and 36 deletions

11
.eslintrc.js Normal file
View File

@@ -0,0 +1,11 @@
module.exports = {
parser: 'babel-eslint',
parserOptions: {
ecmaVersion: 2019,
sourceType: 'module',
},
extends: ['airbnb'],
rules: {
'lines-between-class-members': 0
}
};

1259
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,19 +1,16 @@
{
"name": "maanteeamet-fetch",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "nodemon --watch src --exec babel-node src/index.js"
},
"author": "",
"license": "ISC",
"dependencies": {
"cacheman": "2.2.1",
"cacheman-file": "0.2.1",
"jsdom": "16.2.2",
"node-fetch": "2.6.0",
"puppeteer": "^3.1.0"
"puppeteer": "3.1.0"
},
"devDependencies": {
"@babel/cli": "7.8.4",
@@ -21,6 +18,13 @@
"@babel/node": "7.8.7",
"@babel/plugin-proposal-class-properties": "7.8.3",
"@babel/preset-env": "7.9.6",
"babel-eslint": "^10.1.0",
"eslint": "^6.8.0",
"eslint-config-airbnb": "18.1.0",
"eslint-plugin-import": "2.20.2",
"eslint-plugin-jsx-a11y": "6.2.3",
"eslint-plugin-react": "7.20.0",
"eslint-plugin-react-hooks": "2.5.0",
"nodemon": "2.0.4"
}
}

View File

@@ -1,5 +1,5 @@
import { JSDOM } from "jsdom";
import Car from "../model/Car";
import { JSDOM } from 'jsdom';
import Car from '../model/Car';
import Selectors from '../util/Selectors';
class Scraper {
@@ -9,7 +9,7 @@ class Scraper {
setContent(text) {
const parsedContent = new JSDOM(text).window.document;
if (parsedContent.querySelector(Selectors.properties.main.container) === null) {
throw Error('No data was received. Cookie is probably expired.')
throw Error('No data was received. Cookie is probably expired.');
}
this.document = parsedContent;
}
@@ -20,7 +20,7 @@ class Scraper {
scrapeMainProperties() {
const {
main: selector
main: selector,
} = Selectors.properties;
this.document
.querySelector(selector.container)
@@ -34,25 +34,22 @@ class Scraper {
data = value[1].innerHTML;
}
this.car[value[0].innerHTML] = data;
})
});
}
scrapeBasicProperties() {
if (!this.document) {
throw Error('No data to scrape.');
}
const properties = this.document.querySelector('#content');
const {
properties: selector
properties: selector,
} = Selectors;
const plate = this.getTextBySelector(selector.plate);
const carName = this.getTextBySelector(selector.name);
const vin = this.getTextBySelector(selector.vin);
console.log(plate, carName, vin);
this.car = new Car(plate, carName, vin.substring(5));
return this.car;
}
}
export default Scraper;

View File

@@ -2,7 +2,8 @@ import puppeteer from 'puppeteer';
import {
SEARCH_URL,
NAVIGATION_TIMEOUT,
TEMP_DIR
TEMP_DIR,
SITE_COOKIES,
} from '../util/Constants';
import Selectors from '../util/Selectors';
@@ -16,12 +17,13 @@ class CookieMonster {
}
async submitForm(plate) {
await this.page.focus(Selectors.form.plate)
await this.page.focus(Selectors.form.plate);
await this.page.keyboard.type(plate);
await this.page.evaluate(_ => {
await this.page.evaluate(() => {
// eslint-disable-next-line no-undef
PrimeFaces.ab({
s: "j_idt104:j_idt131",
u: "j_idt104"
s: 'j_idt104:j_idt131',
u: 'j_idt104',
});
});
await this.page.waitForNavigation({
@@ -34,11 +36,13 @@ class CookieMonster {
this.browser = await puppeteer.launch();
this.page = await this.browser.newPage();
await this.page.goto(SEARCH_URL);
await this.page.setCookie(...SITE_COOKIES);
}
async cleanup(plate) {
await this.page.screenshot({
path: `${TEMP_DIR.screenshots}/${plate}.png`
path: `${TEMP_DIR.screenshots}/${plate}.png`,
fullPage: true,
});
await this.browser.close();
}
@@ -46,13 +50,13 @@ class CookieMonster {
async init(plate) {
console.log(`Fetching data for ${plate}`);
await this.launchPage();
await this.submitForm(plate)
await this.submitForm(plate);
const pageContent = await this.page
.$eval(Selectors.container.main, (element) => element.innerHTML);
await this.cleanup(plate);
console.log(`Successfully fetched fresh data for ${plate}`);
return pageContent;
}
};
}
export default CookieMonster;

View File

@@ -5,7 +5,9 @@ import CookieMonster from './cookies/CookieMonster';
class Hack {
scraper;
cache;
cookieMonster;
constructor() {
@@ -15,7 +17,7 @@ class Hack {
}
async getData(plate) {
let cached = await this.cache.get(plate);
const cached = await this.cache.get(plate);
if (cached) {
console.log(`Using cached data for ${plate}`);
return cached;

View File

@@ -1,6 +1,13 @@
import Cacheman from 'cacheman';
import { CACHE } from './Constants';
const formatKey = (name) => {
if (!name) {
throw Error('No number plate specified');
}
return `${CACHE.PREFIX.plate}${name}`;
};
class Cache {
manager;
@@ -8,28 +15,19 @@ class Cache {
this.manager = new Cacheman({
ttl: CACHE.ttl,
engine: CACHE.engine,
tmpDir: CACHE.directory
tmpDir: CACHE.directory,
});
}
formatKey(name) {
if (!name) {
throw Error('No number plate specified');
}
return `${CACHE.PREFIX.plate}${name}`;
}
async get(name) {
const key = this.formatKey(name);
return this.manager.get(key);
return this.manager.get(formatKey(name));
}
save(name, data) {
const key = this.formatKey(name);
if (!data) {
throw Error(`No data for caching car ${name}`);
}
this.manager.set(key, data);
this.manager.set(formatKey(name), data);
}
}

View File

@@ -1,6 +1,11 @@
export const BASE_URL = 'https://eteenindus.mnt.ee/public/soidukDetailvaadeAvalik.jsf';
export const SEARCH_URL = 'https://eteenindus.mnt.ee/public/soidukTaustakontroll.jsf';
export const SITE_COOKIES = [{
name: 'eteenindus_lang',
value: 'en',
}];
export const NAVIGATION_TIMEOUT = 2500;
const TMP_DIR = 'tmp';