some improvements
This commit is contained in:
7032
api/package-lock.json
generated
7032
api/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -7,16 +7,16 @@
|
||||
"dev": "nodemon -r dotenv/config --watch src --exec babel-node src/index.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": "14.3.0",
|
||||
"npm": "6.14.5"
|
||||
"node": "^14",
|
||||
"npm": "^6.14"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/cli": "7.8.4",
|
||||
"@babel/core": "7.9.6",
|
||||
"@babel/node": "7.8.7",
|
||||
"@babel/plugin-proposal-class-properties": "7.8.3",
|
||||
"@babel/preset-env": "7.9.6",
|
||||
"body-parser": "^1.19.0",
|
||||
"@babel/cli": "7.10.4",
|
||||
"@babel/core": "7.10.4",
|
||||
"@babel/node": "7.10.4",
|
||||
"@babel/plugin-proposal-class-properties": "7.10.4",
|
||||
"@babel/preset-env": "7.10.4",
|
||||
"body-parser": "1.19.0",
|
||||
"cacheman": "2.2.1",
|
||||
"cacheman-file": "0.2.1",
|
||||
"compression": "1.7.4",
|
||||
@@ -24,20 +24,20 @@
|
||||
"dotenv": "8.2.0",
|
||||
"express": "4.17.1",
|
||||
"form-data": "3.0.0",
|
||||
"helmet": "3.22.0",
|
||||
"helmet": "3.23.3",
|
||||
"jsdom": "16.2.2",
|
||||
"multer": "^1.4.2",
|
||||
"multer": "1.4.2",
|
||||
"node-fetch": "2.6.0",
|
||||
"puppeteer": "3.1.0"
|
||||
"puppeteer": "5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"babel-eslint": "10.1.0",
|
||||
"eslint": "7.1.0",
|
||||
"eslint-config-airbnb": "18.1.0",
|
||||
"eslint-plugin-import": "2.20.2",
|
||||
"eslint-plugin-jsx-a11y": "6.2.3",
|
||||
"eslint-plugin-react": "7.20.0",
|
||||
"eslint-plugin-react-hooks": "4.0.2",
|
||||
"eslint": "7.4.0",
|
||||
"eslint-config-airbnb": "18.2.0",
|
||||
"eslint-plugin-import": "2.22.0",
|
||||
"eslint-plugin-jsx-a11y": "6.3.1",
|
||||
"eslint-plugin-react": "7.20.3",
|
||||
"eslint-plugin-react-hooks": "4.0.6",
|
||||
"nodemon": "2.0.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
TEMP_DIR,
|
||||
SITE_COOKIES,
|
||||
} from '../util/Constants';
|
||||
import Selectors from '../util/Selectors';
|
||||
import Selectors, { SUBMIT_FORM_ID } from '../util/Selectors';
|
||||
import { createCacheDirectories } from '../util/TempDirCreator';
|
||||
|
||||
const getPuppeteerOptions = () => {
|
||||
@@ -15,7 +15,7 @@ const getPuppeteerOptions = () => {
|
||||
}
|
||||
return options;
|
||||
};
|
||||
|
||||
console.log(Selectors);
|
||||
class CookieMonster {
|
||||
cache;
|
||||
browser;
|
||||
@@ -29,11 +29,15 @@ class CookieMonster {
|
||||
async submitForm(plate) {
|
||||
await this.page.focus(Selectors.form.plate);
|
||||
await this.page.keyboard.type(plate);
|
||||
const path = {
|
||||
s: `j_idt${SUBMIT_FORM_ID}:j_idt137`,
|
||||
u: `j_idt${SUBMIT_FORM_ID}`,
|
||||
};
|
||||
await this.page.evaluate(() => {
|
||||
// eslint-disable-next-line no-undef
|
||||
PrimeFaces.ab({
|
||||
s: 'j_idt104:j_idt131',
|
||||
u: 'j_idt104',
|
||||
s: 'j_idt110:j_idt137',
|
||||
u: 'j_idt110',
|
||||
});
|
||||
});
|
||||
await this.page.waitForNavigation({
|
||||
@@ -58,14 +62,23 @@ class CookieMonster {
|
||||
}
|
||||
|
||||
async init(plate) {
|
||||
console.log(`Fetching data for ${plate}`);
|
||||
await this.launchPage();
|
||||
await this.submitForm(plate);
|
||||
const pageContent = await this.page
|
||||
.$eval(Selectors.container.main, (element) => element.innerHTML);
|
||||
await this.cleanup(plate);
|
||||
console.log(`Successfully fetched fresh data for ${plate}`);
|
||||
return pageContent;
|
||||
try {
|
||||
console.log(`Fetching data for ${plate}`);
|
||||
await this.launchPage();
|
||||
try {
|
||||
await this.submitForm(plate);
|
||||
} catch (e) {
|
||||
console.warn(`Got a timeout for ${plate} but data might still be there, sometimes takes longer to load entire page.`, e.message);
|
||||
}
|
||||
const pageContent = await this.page
|
||||
.$eval(Selectors.container.main, (element) => element.innerHTML);
|
||||
await this.cleanup(plate);
|
||||
console.log(`Successfully fetched fresh data for ${plate}`);
|
||||
return pageContent;
|
||||
} catch (e) {
|
||||
await this.cleanup(plate);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ class Hack {
|
||||
async init(plate) {
|
||||
try {
|
||||
const data = await this.getData(plate);
|
||||
console.log(data);
|
||||
this.scraper.setContent(data);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
||||
@@ -9,7 +9,7 @@ class Scraper {
|
||||
setContent(text) {
|
||||
const parsedContent = new JSDOM(text).window.document;
|
||||
if (parsedContent.querySelector(Selectors.properties.main.container) === null) {
|
||||
throw Error('No data was received.. Something went wrong.');
|
||||
throw Error('No data or invalid data was received.. Something went wrong.');
|
||||
}
|
||||
this.document = parsedContent;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
// seems that this changes sometimes
|
||||
export const SUBMIT_FORM_ID = '110';
|
||||
|
||||
export default {
|
||||
form: {
|
||||
plate: '#j_idt104\\:regMark',
|
||||
plate: `#j_idt${SUBMIT_FORM_ID}\\:regMark`,
|
||||
},
|
||||
container: {
|
||||
main: '#content',
|
||||
form: '#j_idt104',
|
||||
form: `#j_idt${SUBMIT_FORM_ID}`,
|
||||
},
|
||||
properties: {
|
||||
plate: '.content-title h1',
|
||||
|
||||
5283
api/yarn.lock
Normal file
5283
api/yarn.lock
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user