added eslint, some cleanup and fetch in english
This commit is contained in:
11
.eslintrc.js
Normal file
11
.eslintrc.js
Normal file
@@ -0,0 +1,11 @@
|
||||
module.exports = {
|
||||
parser: 'babel-eslint',
|
||||
parserOptions: {
|
||||
ecmaVersion: 2019,
|
||||
sourceType: 'module',
|
||||
},
|
||||
extends: ['airbnb'],
|
||||
rules: {
|
||||
'lines-between-class-members': 0
|
||||
}
|
||||
};
|
||||
1259
package-lock.json
generated
1259
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
12
package.json
12
package.json
@@ -1,19 +1,16 @@
|
||||
{
|
||||
"name": "maanteeamet-fetch",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "nodemon --watch src --exec babel-node src/index.js"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cacheman": "2.2.1",
|
||||
"cacheman-file": "0.2.1",
|
||||
"jsdom": "16.2.2",
|
||||
"node-fetch": "2.6.0",
|
||||
"puppeteer": "^3.1.0"
|
||||
"puppeteer": "3.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "7.8.4",
|
||||
@@ -21,6 +18,13 @@
|
||||
"@babel/node": "7.8.7",
|
||||
"@babel/plugin-proposal-class-properties": "7.8.3",
|
||||
"@babel/preset-env": "7.9.6",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"eslint": "^6.8.0",
|
||||
"eslint-config-airbnb": "18.1.0",
|
||||
"eslint-plugin-import": "2.20.2",
|
||||
"eslint-plugin-jsx-a11y": "6.2.3",
|
||||
"eslint-plugin-react": "7.20.0",
|
||||
"eslint-plugin-react-hooks": "2.5.0",
|
||||
"nodemon": "2.0.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
import Car from "../model/Car";
|
||||
import { JSDOM } from 'jsdom';
|
||||
import Car from '../model/Car';
|
||||
import Selectors from '../util/Selectors';
|
||||
|
||||
class Scraper {
|
||||
@@ -9,7 +9,7 @@ class Scraper {
|
||||
setContent(text) {
|
||||
const parsedContent = new JSDOM(text).window.document;
|
||||
if (parsedContent.querySelector(Selectors.properties.main.container) === null) {
|
||||
throw Error('No data was received. Cookie is probably expired.')
|
||||
throw Error('No data was received. Cookie is probably expired.');
|
||||
}
|
||||
this.document = parsedContent;
|
||||
}
|
||||
@@ -20,7 +20,7 @@ class Scraper {
|
||||
|
||||
scrapeMainProperties() {
|
||||
const {
|
||||
main: selector
|
||||
main: selector,
|
||||
} = Selectors.properties;
|
||||
this.document
|
||||
.querySelector(selector.container)
|
||||
@@ -34,25 +34,22 @@ class Scraper {
|
||||
data = value[1].innerHTML;
|
||||
}
|
||||
this.car[value[0].innerHTML] = data;
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
scrapeBasicProperties() {
|
||||
if (!this.document) {
|
||||
throw Error('No data to scrape.');
|
||||
}
|
||||
const properties = this.document.querySelector('#content');
|
||||
const {
|
||||
properties: selector
|
||||
properties: selector,
|
||||
} = Selectors;
|
||||
const plate = this.getTextBySelector(selector.plate);
|
||||
const carName = this.getTextBySelector(selector.name);
|
||||
const vin = this.getTextBySelector(selector.vin);
|
||||
console.log(plate, carName, vin);
|
||||
this.car = new Car(plate, carName, vin.substring(5));
|
||||
return this.car;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default Scraper;
|
||||
|
||||
@@ -2,7 +2,8 @@ import puppeteer from 'puppeteer';
|
||||
import {
|
||||
SEARCH_URL,
|
||||
NAVIGATION_TIMEOUT,
|
||||
TEMP_DIR
|
||||
TEMP_DIR,
|
||||
SITE_COOKIES,
|
||||
} from '../util/Constants';
|
||||
import Selectors from '../util/Selectors';
|
||||
|
||||
@@ -16,12 +17,13 @@ class CookieMonster {
|
||||
}
|
||||
|
||||
async submitForm(plate) {
|
||||
await this.page.focus(Selectors.form.plate)
|
||||
await this.page.focus(Selectors.form.plate);
|
||||
await this.page.keyboard.type(plate);
|
||||
await this.page.evaluate(_ => {
|
||||
await this.page.evaluate(() => {
|
||||
// eslint-disable-next-line no-undef
|
||||
PrimeFaces.ab({
|
||||
s: "j_idt104:j_idt131",
|
||||
u: "j_idt104"
|
||||
s: 'j_idt104:j_idt131',
|
||||
u: 'j_idt104',
|
||||
});
|
||||
});
|
||||
await this.page.waitForNavigation({
|
||||
@@ -34,11 +36,13 @@ class CookieMonster {
|
||||
this.browser = await puppeteer.launch();
|
||||
this.page = await this.browser.newPage();
|
||||
await this.page.goto(SEARCH_URL);
|
||||
await this.page.setCookie(...SITE_COOKIES);
|
||||
}
|
||||
|
||||
async cleanup(plate) {
|
||||
await this.page.screenshot({
|
||||
path: `${TEMP_DIR.screenshots}/${plate}.png`
|
||||
path: `${TEMP_DIR.screenshots}/${plate}.png`,
|
||||
fullPage: true,
|
||||
});
|
||||
await this.browser.close();
|
||||
}
|
||||
@@ -46,13 +50,13 @@ class CookieMonster {
|
||||
async init(plate) {
|
||||
console.log(`Fetching data for ${plate}`);
|
||||
await this.launchPage();
|
||||
await this.submitForm(plate)
|
||||
await this.submitForm(plate);
|
||||
const pageContent = await this.page
|
||||
.$eval(Selectors.container.main, (element) => element.innerHTML);
|
||||
await this.cleanup(plate);
|
||||
console.log(`Successfully fetched fresh data for ${plate}`);
|
||||
return pageContent;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default CookieMonster;
|
||||
|
||||
@@ -5,7 +5,9 @@ import CookieMonster from './cookies/CookieMonster';
|
||||
|
||||
class Hack {
|
||||
scraper;
|
||||
|
||||
cache;
|
||||
|
||||
cookieMonster;
|
||||
|
||||
constructor() {
|
||||
@@ -15,7 +17,7 @@ class Hack {
|
||||
}
|
||||
|
||||
async getData(plate) {
|
||||
let cached = await this.cache.get(plate);
|
||||
const cached = await this.cache.get(plate);
|
||||
if (cached) {
|
||||
console.log(`Using cached data for ${plate}`);
|
||||
return cached;
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import Cacheman from 'cacheman';
|
||||
import { CACHE } from './Constants';
|
||||
|
||||
const formatKey = (name) => {
|
||||
if (!name) {
|
||||
throw Error('No number plate specified');
|
||||
}
|
||||
return `${CACHE.PREFIX.plate}${name}`;
|
||||
};
|
||||
|
||||
class Cache {
|
||||
manager;
|
||||
|
||||
@@ -8,28 +15,19 @@ class Cache {
|
||||
this.manager = new Cacheman({
|
||||
ttl: CACHE.ttl,
|
||||
engine: CACHE.engine,
|
||||
tmpDir: CACHE.directory
|
||||
tmpDir: CACHE.directory,
|
||||
});
|
||||
}
|
||||
|
||||
formatKey(name) {
|
||||
if (!name) {
|
||||
throw Error('No number plate specified');
|
||||
}
|
||||
return `${CACHE.PREFIX.plate}${name}`;
|
||||
}
|
||||
|
||||
async get(name) {
|
||||
const key = this.formatKey(name);
|
||||
return this.manager.get(key);
|
||||
return this.manager.get(formatKey(name));
|
||||
}
|
||||
|
||||
save(name, data) {
|
||||
const key = this.formatKey(name);
|
||||
if (!data) {
|
||||
throw Error(`No data for caching car ${name}`);
|
||||
}
|
||||
this.manager.set(key, data);
|
||||
this.manager.set(formatKey(name), data);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
export const BASE_URL = 'https://eteenindus.mnt.ee/public/soidukDetailvaadeAvalik.jsf';
|
||||
export const SEARCH_URL = 'https://eteenindus.mnt.ee/public/soidukTaustakontroll.jsf';
|
||||
|
||||
export const SITE_COOKIES = [{
|
||||
name: 'eteenindus_lang',
|
||||
value: 'en',
|
||||
}];
|
||||
|
||||
export const NAVIGATION_TIMEOUT = 2500;
|
||||
|
||||
const TMP_DIR = 'tmp';
|
||||
|
||||
Reference in New Issue
Block a user