some improvements

This commit is contained in:
2020-07-08 11:44:05 +03:00
parent 167a608b93
commit b2eae74b9e
23 changed files with 15122 additions and 21816 deletions

7032
api/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,16 +7,16 @@
"dev": "nodemon -r dotenv/config --watch src --exec babel-node src/index.js"
},
"engines": {
"node": "14.3.0",
"npm": "6.14.5"
"node": "^14",
"npm": "^6.14"
},
"dependencies": {
"@babel/cli": "7.8.4",
"@babel/core": "7.9.6",
"@babel/node": "7.8.7",
"@babel/plugin-proposal-class-properties": "7.8.3",
"@babel/preset-env": "7.9.6",
"body-parser": "^1.19.0",
"@babel/cli": "7.10.4",
"@babel/core": "7.10.4",
"@babel/node": "7.10.4",
"@babel/plugin-proposal-class-properties": "7.10.4",
"@babel/preset-env": "7.10.4",
"body-parser": "1.19.0",
"cacheman": "2.2.1",
"cacheman-file": "0.2.1",
"compression": "1.7.4",
@@ -24,20 +24,20 @@
"dotenv": "8.2.0",
"express": "4.17.1",
"form-data": "3.0.0",
"helmet": "3.22.0",
"helmet": "3.23.3",
"jsdom": "16.2.2",
"multer": "^1.4.2",
"multer": "1.4.2",
"node-fetch": "2.6.0",
"puppeteer": "3.1.0"
"puppeteer": "5.0.0"
},
"devDependencies": {
"babel-eslint": "10.1.0",
"eslint": "7.1.0",
"eslint-config-airbnb": "18.1.0",
"eslint-plugin-import": "2.20.2",
"eslint-plugin-jsx-a11y": "6.2.3",
"eslint-plugin-react": "7.20.0",
"eslint-plugin-react-hooks": "4.0.2",
"eslint": "7.4.0",
"eslint-config-airbnb": "18.2.0",
"eslint-plugin-import": "2.22.0",
"eslint-plugin-jsx-a11y": "6.3.1",
"eslint-plugin-react": "7.20.3",
"eslint-plugin-react-hooks": "4.0.6",
"nodemon": "2.0.4"
}
}

View File

@@ -5,7 +5,7 @@ import {
TEMP_DIR,
SITE_COOKIES,
} from '../util/Constants';
import Selectors from '../util/Selectors';
import Selectors, { SUBMIT_FORM_ID } from '../util/Selectors';
import { createCacheDirectories } from '../util/TempDirCreator';
const getPuppeteerOptions = () => {
@@ -15,7 +15,7 @@ const getPuppeteerOptions = () => {
}
return options;
};
console.log(Selectors);
class CookieMonster {
cache;
browser;
@@ -29,11 +29,15 @@ class CookieMonster {
async submitForm(plate) {
await this.page.focus(Selectors.form.plate);
await this.page.keyboard.type(plate);
const path = {
s: `j_idt${SUBMIT_FORM_ID}:j_idt137`,
u: `j_idt${SUBMIT_FORM_ID}`,
};
await this.page.evaluate(() => {
// eslint-disable-next-line no-undef
PrimeFaces.ab({
s: 'j_idt104:j_idt131',
u: 'j_idt104',
s: 'j_idt110:j_idt137',
u: 'j_idt110',
});
});
await this.page.waitForNavigation({
@@ -58,14 +62,23 @@ class CookieMonster {
}
async init(plate) {
console.log(`Fetching data for ${plate}`);
await this.launchPage();
await this.submitForm(plate);
const pageContent = await this.page
.$eval(Selectors.container.main, (element) => element.innerHTML);
await this.cleanup(plate);
console.log(`Successfully fetched fresh data for ${plate}`);
return pageContent;
try {
console.log(`Fetching data for ${plate}`);
await this.launchPage();
try {
await this.submitForm(plate);
} catch (e) {
console.warn(`Got a timeout for ${plate} but data might still be there, sometimes takes longer to load entire page.`, e.message);
}
const pageContent = await this.page
.$eval(Selectors.container.main, (element) => element.innerHTML);
await this.cleanup(plate);
console.log(`Successfully fetched fresh data for ${plate}`);
return pageContent;
} catch (e) {
await this.cleanup(plate);
throw e;
}
}
}

View File

@@ -28,6 +28,7 @@ class Hack {
async init(plate) {
try {
const data = await this.getData(plate);
console.log(data);
this.scraper.setContent(data);
} catch (e) {
console.error(e);

View File

@@ -9,7 +9,7 @@ class Scraper {
setContent(text) {
const parsedContent = new JSDOM(text).window.document;
if (parsedContent.querySelector(Selectors.properties.main.container) === null) {
throw Error('No data was received.. Something went wrong.');
throw Error('No data or invalid data was received.. Something went wrong.');
}
this.document = parsedContent;
}

View File

@@ -1,10 +1,13 @@
// seems that this changes sometimes
export const SUBMIT_FORM_ID = '110';
export default {
form: {
plate: '#j_idt104\\:regMark',
plate: `#j_idt${SUBMIT_FORM_ID}\\:regMark`,
},
container: {
main: '#content',
form: '#j_idt104',
form: `#j_idt${SUBMIT_FORM_ID}`,
},
properties: {
plate: '.content-title h1',

5283
api/yarn.lock Normal file

File diff suppressed because it is too large Load Diff