some improvements

This commit is contained in:
2020-07-08 11:44:05 +03:00
parent 167a608b93
commit b2eae74b9e
23 changed files with 15122 additions and 21816 deletions

7032
api/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,16 +7,16 @@
"dev": "nodemon -r dotenv/config --watch src --exec babel-node src/index.js" "dev": "nodemon -r dotenv/config --watch src --exec babel-node src/index.js"
}, },
"engines": { "engines": {
"node": "14.3.0", "node": "^14",
"npm": "6.14.5" "npm": "^6.14"
}, },
"dependencies": { "dependencies": {
"@babel/cli": "7.8.4", "@babel/cli": "7.10.4",
"@babel/core": "7.9.6", "@babel/core": "7.10.4",
"@babel/node": "7.8.7", "@babel/node": "7.10.4",
"@babel/plugin-proposal-class-properties": "7.8.3", "@babel/plugin-proposal-class-properties": "7.10.4",
"@babel/preset-env": "7.9.6", "@babel/preset-env": "7.10.4",
"body-parser": "^1.19.0", "body-parser": "1.19.0",
"cacheman": "2.2.1", "cacheman": "2.2.1",
"cacheman-file": "0.2.1", "cacheman-file": "0.2.1",
"compression": "1.7.4", "compression": "1.7.4",
@@ -24,20 +24,20 @@
"dotenv": "8.2.0", "dotenv": "8.2.0",
"express": "4.17.1", "express": "4.17.1",
"form-data": "3.0.0", "form-data": "3.0.0",
"helmet": "3.22.0", "helmet": "3.23.3",
"jsdom": "16.2.2", "jsdom": "16.2.2",
"multer": "^1.4.2", "multer": "1.4.2",
"node-fetch": "2.6.0", "node-fetch": "2.6.0",
"puppeteer": "3.1.0" "puppeteer": "5.0.0"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "10.1.0", "babel-eslint": "10.1.0",
"eslint": "7.1.0", "eslint": "7.4.0",
"eslint-config-airbnb": "18.1.0", "eslint-config-airbnb": "18.2.0",
"eslint-plugin-import": "2.20.2", "eslint-plugin-import": "2.22.0",
"eslint-plugin-jsx-a11y": "6.2.3", "eslint-plugin-jsx-a11y": "6.3.1",
"eslint-plugin-react": "7.20.0", "eslint-plugin-react": "7.20.3",
"eslint-plugin-react-hooks": "4.0.2", "eslint-plugin-react-hooks": "4.0.6",
"nodemon": "2.0.4" "nodemon": "2.0.4"
} }
} }

View File

@@ -5,7 +5,7 @@ import {
TEMP_DIR, TEMP_DIR,
SITE_COOKIES, SITE_COOKIES,
} from '../util/Constants'; } from '../util/Constants';
import Selectors from '../util/Selectors'; import Selectors, { SUBMIT_FORM_ID } from '../util/Selectors';
import { createCacheDirectories } from '../util/TempDirCreator'; import { createCacheDirectories } from '../util/TempDirCreator';
const getPuppeteerOptions = () => { const getPuppeteerOptions = () => {
@@ -15,7 +15,7 @@ const getPuppeteerOptions = () => {
} }
return options; return options;
}; };
console.log(Selectors);
class CookieMonster { class CookieMonster {
cache; cache;
browser; browser;
@@ -29,11 +29,15 @@ class CookieMonster {
async submitForm(plate) { async submitForm(plate) {
await this.page.focus(Selectors.form.plate); await this.page.focus(Selectors.form.plate);
await this.page.keyboard.type(plate); await this.page.keyboard.type(plate);
const path = {
s: `j_idt${SUBMIT_FORM_ID}:j_idt137`,
u: `j_idt${SUBMIT_FORM_ID}`,
};
await this.page.evaluate(() => { await this.page.evaluate(() => {
// eslint-disable-next-line no-undef // eslint-disable-next-line no-undef
PrimeFaces.ab({ PrimeFaces.ab({
s: 'j_idt104:j_idt131', s: 'j_idt110:j_idt137',
u: 'j_idt104', u: 'j_idt110',
}); });
}); });
await this.page.waitForNavigation({ await this.page.waitForNavigation({
@@ -58,14 +62,23 @@ class CookieMonster {
} }
async init(plate) { async init(plate) {
console.log(`Fetching data for ${plate}`); try {
await this.launchPage(); console.log(`Fetching data for ${plate}`);
await this.submitForm(plate); await this.launchPage();
const pageContent = await this.page try {
.$eval(Selectors.container.main, (element) => element.innerHTML); await this.submitForm(plate);
await this.cleanup(plate); } catch (e) {
console.log(`Successfully fetched fresh data for ${plate}`); console.warn(`Got a timeout for ${plate} but data might still be there, sometimes takes longer to load entire page.`, e.message);
return pageContent; }
const pageContent = await this.page
.$eval(Selectors.container.main, (element) => element.innerHTML);
await this.cleanup(plate);
console.log(`Successfully fetched fresh data for ${plate}`);
return pageContent;
} catch (e) {
await this.cleanup(plate);
throw e;
}
} }
} }

View File

@@ -28,6 +28,7 @@ class Hack {
async init(plate) { async init(plate) {
try { try {
const data = await this.getData(plate); const data = await this.getData(plate);
console.log(data);
this.scraper.setContent(data); this.scraper.setContent(data);
} catch (e) { } catch (e) {
console.error(e); console.error(e);

View File

@@ -9,7 +9,7 @@ class Scraper {
setContent(text) { setContent(text) {
const parsedContent = new JSDOM(text).window.document; const parsedContent = new JSDOM(text).window.document;
if (parsedContent.querySelector(Selectors.properties.main.container) === null) { if (parsedContent.querySelector(Selectors.properties.main.container) === null) {
throw Error('No data was received.. Something went wrong.'); throw Error('No data or invalid data was received.. Something went wrong.');
} }
this.document = parsedContent; this.document = parsedContent;
} }

View File

@@ -1,10 +1,13 @@
// seems that this changes sometimes
export const SUBMIT_FORM_ID = '110';
export default { export default {
form: { form: {
plate: '#j_idt104\\:regMark', plate: `#j_idt${SUBMIT_FORM_ID}\\:regMark`,
}, },
container: { container: {
main: '#content', main: '#content',
form: '#j_idt104', form: `#j_idt${SUBMIT_FORM_ID}`,
}, },
properties: { properties: {
plate: '.content-title h1', plate: '.content-title h1',

5283
api/yarn.lock Normal file

File diff suppressed because it is too large Load Diff

14692
frontend/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -12,8 +12,8 @@
"lint": "eslint --fix src --ext .jsx" "lint": "eslint --fix src --ext .jsx"
}, },
"engines": { "engines": {
"node": "14.3.0", "node": "^14",
"npm": "6.14.5" "npm": "^6.14"
}, },
"lint-staged": { "lint-staged": {
"*.{js,jsx,css,md}": [ "*.{js,jsx,css,md}": [
@@ -22,68 +22,70 @@
] ]
}, },
"dependencies": { "dependencies": {
"@fortawesome/fontawesome-free": "5.13.0", "@fortawesome/fontawesome-free": "5.13.1",
"@reduxjs/toolkit": "^1.3.6", "@material-ui/core": "4.11.0",
"@material-ui/icons": "4.9.1",
"@reduxjs/toolkit": "1.4.0",
"core-js": "3.6.5", "core-js": "3.6.5",
"react": "16.13.1", "react": "16.13.1",
"react-dom": "16.13.1", "react-dom": "16.13.1",
"react-hot-loader": "4.12.21", "react-hot-loader": "4.12.21",
"react-webcam": "5.0.2", "react-webcam": "5.2.0",
"regenerator-runtime": "0.13.5", "regenerator-runtime": "0.13.5",
"use-dark-mode": "2.3.1" "use-dark-mode": "2.3.1"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "7.9.6", "@babel/core": "7.10.4",
"@babel/plugin-proposal-class-properties": "7.8.3", "@babel/plugin-proposal-class-properties": "7.10.4",
"@babel/plugin-proposal-decorators": "7.8.3", "@babel/plugin-proposal-decorators": "7.10.4",
"@babel/plugin-proposal-export-namespace-from": "7.8.3", "@babel/plugin-proposal-export-namespace-from": "7.10.4",
"@babel/plugin-proposal-function-sent": "7.8.3", "@babel/plugin-proposal-function-sent": "7.10.4",
"@babel/plugin-proposal-json-strings": "7.8.3", "@babel/plugin-proposal-json-strings": "7.10.4",
"@babel/plugin-proposal-numeric-separator": "7.8.3", "@babel/plugin-proposal-numeric-separator": "7.10.4",
"@babel/plugin-proposal-optional-chaining": "7.9.0", "@babel/plugin-proposal-optional-chaining": "7.10.4",
"@babel/plugin-proposal-throw-expressions": "7.8.3", "@babel/plugin-proposal-throw-expressions": "7.10.4",
"@babel/plugin-syntax-dynamic-import": "7.8.3", "@babel/plugin-syntax-dynamic-import": "7.8.3",
"@babel/plugin-syntax-import-meta": "7.8.3", "@babel/plugin-syntax-import-meta": "7.10.4",
"@babel/plugin-transform-runtime": "7.9.6", "@babel/plugin-transform-runtime": "7.10.4",
"@babel/preset-env": "7.9.6", "@babel/preset-env": "7.10.4",
"@babel/preset-react": "7.9.4", "@babel/preset-react": "7.10.4",
"@babel/register": "7.9.0", "@babel/register": "7.10.4",
"@babel/runtime-corejs3": "7.9.6", "@babel/runtime-corejs3": "7.10.4",
"babel-eslint": "10.1.0", "babel-eslint": "10.1.0",
"babel-loader": "8.1.0", "babel-loader": "8.1.0",
"babel-plugin-module-resolver": "4.0.0", "babel-plugin-module-resolver": "4.0.0",
"browserslist": "4.12.0", "browserslist": "4.13.0",
"connect-history-api-fallback": "1.6.0", "connect-history-api-fallback": "1.6.0",
"copy-webpack-plugin": "6.0.1", "copy-webpack-plugin": "6.0.3",
"cross-env": "7.0.2", "cross-env": "7.0.2",
"css-loader": "3.5.3", "css-loader": "3.6.0",
"dotenv": "^8.2.0", "dotenv": "8.2.0",
"eslint": "7.1.0", "eslint": "7.4.0",
"eslint-config-airbnb": "18.1.0", "eslint-config-airbnb": "18.2.0",
"eslint-import-resolver-babel-module": "5.1.2", "eslint-import-resolver-babel-module": "5.1.2",
"eslint-loader": "4.0.2", "eslint-loader": "4.0.2",
"eslint-plugin-import": "2.20.2", "eslint-plugin-import": "2.22.0",
"eslint-plugin-jsx-a11y": "6.2.3", "eslint-plugin-jsx-a11y": "6.3.1",
"eslint-plugin-react": "7.20.0", "eslint-plugin-react": "7.20.3",
"eslint-plugin-react-hooks": "4.0.2", "eslint-plugin-react-hooks": "4.0.6",
"file-loader": "6.0.0", "file-loader": "6.0.0",
"hard-source-webpack-plugin": "0.13.1", "hard-source-webpack-plugin": "0.13.1",
"html-webpack-plugin": "4.3.0", "html-webpack-plugin": "4.3.0",
"koa-connect": "2.0.1", "koa-connect": "2.1.0",
"lint-staged": "10.2.6", "lint-staged": "10.2.11",
"mini-css-extract-plugin": "0.9.0", "mini-css-extract-plugin": "0.9.0",
"node-sass": "4.14.1", "node-sass": "4.14.1",
"nodemon": "2.0.4", "nodemon": "2.0.4",
"optimize-css-assets-webpack-plugin": "5.0.3", "optimize-css-assets-webpack-plugin": "5.0.3",
"rimraf": "3.0.2", "rimraf": "3.0.2",
"sass-loader": "8.0.2", "sass-loader": "9.0.2",
"script-ext-html-webpack-plugin": "2.1.4", "script-ext-html-webpack-plugin": "2.1.4",
"style-loader": "1.2.1", "style-loader": "1.2.1",
"terser-webpack-plugin": "3.0.1", "terser-webpack-plugin": "3.0.6",
"webpack": "4.43.0", "webpack": "4.43.0",
"webpack-bundle-analyzer": "3.8.0", "webpack-bundle-analyzer": "3.8.0",
"webpack-cli": "3.3.11", "webpack-cli": "3.3.12",
"webpack-dev-server": "3.11.0", "webpack-dev-server": "3.11.0",
"webpack-merge": "4.2.2" "webpack-merge": "5.0.8"
} }
} }

View File

@@ -5,7 +5,6 @@
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>&#x1F615;</title> <title>&#x1F615;</title>
<link type="text/css" rel="stylesheet" href="./assets/css/skeleton.min.css" /> <link type="text/css" rel="stylesheet" href="./assets/css/skeleton.min.css" />
<link type="text/css" rel="stylesheet" href="https://fonts.googleapis.com/css?family=Inconsolata&display=swap">
<link rel="apple-touch-icon" sizes="180x180" href="./assets/images/icon.png"> <link rel="apple-touch-icon" sizes="180x180" href="./assets/images/icon.png">
<link rel="icon" type="image/png" sizes="64x64" href="./assets/images/icon.png"> <link rel="icon" type="image/png" sizes="64x64" href="./assets/images/icon.png">
</head> </head>

View File

@@ -1,17 +1,16 @@
import { import {
alprFetchSuccess, alprFetchStart, alprFetchSuccess, alprFetchStart, dataFetchStart, dataFetchSuccess,
} from '@slice/RecognitionSlice'; } from '@slice/RecognitionSlice';
const API_BASE_PATH = process.env.NODE_ENV === 'production' const API_BASE_PATH = process.env.API_BASE_PATH;
? process.env.API_BASE_PATH console.log(process.env.API_BASE_PATH);
: 'http://localhost:4000';
export const getPlateRecognized = (image) => async (dispatch) => { export const getPlateRecognized = (image) => async (dispatch) => {
console.log(image); console.log(image);
dispatch(alprFetchStart()); dispatch(alprFetchStart());
const body = new FormData(); const body = new FormData();
body.append('upload', image); body.append('upload', image);
try { try {
console.log(API_BASE_PATH);
const response = await fetch(`${API_BASE_PATH}/alpr`, { const response = await fetch(`${API_BASE_PATH}/alpr`, {
method: 'POST', method: 'POST',
body, body,
@@ -25,4 +24,20 @@ export const getPlateRecognized = (image) => async (dispatch) => {
} }
}; };
export const fetchDataForPlate = (plate) => async (dispatch) => {
dispatch(dataFetchStart());
console.log('Fetching data for plate', plate);
try {
const response = await fetch(`${API_BASE_PATH}/${plate}`, {
method: 'GET',
});
console.log(response);
const data = await response.json();
dispatch(dataFetchSuccess(data));
console.log(data);
} catch (e) {
console.log(e);
}
};
export const getPlateData = (plate) => null; export const getPlateData = (plate) => null;

View File

@@ -0,0 +1,25 @@
import React, { useContext } from 'react';
import RootContext from '@context/RootContext';
import { VIEW } from '@slice/StateSlice';
import { fetchDataForPlate } from '@actions/CaptureActions';
import Button from './common/Button';
const BottomNavigation = () => {
const { state: { state, alpr: { results } }, dispatch } = useContext(RootContext);
switch (state.view) {
case VIEW.alprResults:
return (
<>
<div className="row">
<Button text="Fetch data" onClick={() => fetchDataForPlate(results[0].plate)(dispatch)} />
</div>
</>
);
case VIEW.camera:
default:
return null;
}
};
export default BottomNavigation;

View File

@@ -0,0 +1,34 @@
import React, { useContext } from 'react';
import RootContext from '@context/RootContext';
const DataView = () => {
const { state, dispatch } = useContext(RootContext);
const { results, lastFetchedDataPlate } = state.alpr;
const { plate, score } = results;
return (
<div className="row">
<h2>{plate.toUpperCase()}</h2>
<h3>
Confidence
{' '}
{score}
</h3>
{lastFetchedDataPlate === plate && (
<ul>
{Object.entries(results).map(([key, value]) => (
<li key={`${key}`}>
<b>{key}</b>
{' '}
:
{' '}
{value}
</li>
))}
</ul>
)}
</div>
);
};
export default DataView;

View File

@@ -1,28 +1,26 @@
import React, { useContext } from 'react'; import React, { useContext } from 'react';
import RootContext from '@context/RootContext'; import RootContext from '@context/RootContext';
import Button from '@components/common/Button'; import Button from '@components/common/Button';
import { fetchDataForPlate } from '@actions/CaptureActions';
import DataView from './DataView';
const RecognitionResults = () => { const RecognitionResults = () => {
const { state: { alpr: { results } } } = useContext(RootContext); const { state, dispatch } = useContext(RootContext);
const { results, lastFetchedDataPlate } = state.alpr;
const fetchData = (plate) => plate && fetchDataForPlate(plate)(dispatch);
console.log(results);
const getPlate = () => { const getPlate = () => {
if (results.length === 0) { if (!results || !results.plate) {
return (<h2>No plate detected</h2>); return (<h2>No plate detected</h2>);
} }
console.log(results);
const { score, plate } = results[0];
return ( return (
<> <>
<DataView />
<div className="row"> <div className="row">
<h2>{plate.toUpperCase()}</h2> <Button text="Fetch data" onClick={() => fetchData(results.plate)} />
<h3>
Confidence
{' '}
{score}
</h3>
</div>
<div className="row">
<Button text="Fetch data" />
</div> </div>
</> </>
); );

View File

@@ -6,7 +6,7 @@ import RootContext from '@context/RootContext';
import RecognitionResults from '@components/alpr/RecognitionResults'; import RecognitionResults from '@components/alpr/RecognitionResults';
const UploadCapture = ({ imgSrc, onBackClick }) => { const UploadCapture = ({ imgSrc, onBackClick }) => {
const [hasFetched, setHasFetched] = useState(false); const [hasFetched, setHasFetched] = useState(true);
const { state: { alpr: { isFetchingALPR } }, dispatch } = useContext(RootContext); const { state: { alpr: { isFetchingALPR } }, dispatch } = useContext(RootContext);
const onUploadClick = () => { const onUploadClick = () => {
getPlateRecognized(imgSrc)(dispatch); getPlateRecognized(imgSrc)(dispatch);

View File

@@ -1,11 +1,13 @@
import React from 'react'; import React from 'react';
import CameraContainer from '@components/camera/CameraContainer'; import CameraContainer from '@components/camera/CameraContainer';
import '@style/RootContainer.scss'; import '@style/RootContainer.scss';
import BottomNavigation from '@components/BottomNavigation';
const RootContainer = () => ( const RootContainer = () => (
<div className="container root-container"> <div className="container root-container">
<div className="header" /> <div className="header" />
<CameraContainer /> <CameraContainer />
<BottomNavigation />
</div> </div>
); );

View File

@@ -1,6 +1,8 @@
import { createContext } from 'react'; import { createContext } from 'react';
import { initialState as alpr } from '@slice/RecognitionSlice'; import { initialState as alpr } from '@slice/RecognitionSlice';
import { initialState as state } from '@slice/StateSlice';
export default createContext({ export default createContext({
alpr, alpr,
state,
}); });

View File

@@ -1,9 +1,28 @@
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
import { createSlice } from '@reduxjs/toolkit'; import {
createSlice,
} from '@reduxjs/toolkit';
export const initialState = { export const initialState = {
isFetchingALPR: false, isFetchingALPR: false,
results: [], isFetchingData: false,
lastFetchedDataPlate: '540BLG',
results: {
plate: '540BLG',
name: 'ALFA ROMEO 159 SPORTWAGON',
vin: 'ZAR93900007055169',
'first registration': '07.09.2006',
category: 'passenger car',
body: 'universal',
'body colour': 'grey',
engine: '2387 cm',
'engine power': '147 kW',
fuel: 'Diesel',
transmission: 'Manual',
drivetrain: 'front wheel drive',
'registration certificate': 'EL573054',
score: 1.0,
},
}; };
const recognitionSlice = createSlice({ const recognitionSlice = createSlice({
@@ -13,12 +32,25 @@ const recognitionSlice = createSlice({
alprFetchStart(state) { alprFetchStart(state) {
state.isFetchingALPR = true; state.isFetchingALPR = true;
}, },
alprFetchSuccess(state, { payload }) { alprFetchSuccess(state, {
payload,
}) {
state.isFetchingALPR = false; state.isFetchingALPR = false;
state.results = payload.results; state.results = payload.results;
}, },
resetResults(state) { resetResults(state) {
state.results = []; state.results = [];
state.lastFetchedDataPlate = null;
},
dataFetchStart(state) {
state.isFetchingData = true;
},
dataFetchSuccess(state, {
payload,
}) {
state.isFetchingData = false;
state.results = { ...state.results, ...payload };
state.lastFetchedDataPlate = payload ? payload.plate : null;
}, },
}, },
}); });
@@ -27,6 +59,8 @@ export const {
alprFetchStart, alprFetchStart,
alprFetchSuccess, alprFetchSuccess,
resetResults, resetResults,
dataFetchStart,
dataFetchSuccess,
} = recognitionSlice.actions; } = recognitionSlice.actions;
export default recognitionSlice.reducer; export default recognitionSlice.reducer;

View File

@@ -1,6 +1,8 @@
import { combineReducers } from '@reduxjs/toolkit'; import { combineReducers } from '@reduxjs/toolkit';
import alpr from '@slice/RecognitionSlice'; import alpr from '@slice/RecognitionSlice';
import state from '@slice/StateSlice';
export default combineReducers({ export default combineReducers({
alpr, alpr,
state,
}); });

View File

@@ -0,0 +1,29 @@
/* eslint-disable no-param-reassign */
import { createSlice } from '@reduxjs/toolkit';
export const VIEW = {
camera: 'camera',
capture: 'capture',
alprResults: 'alprResults',
dataResults: 'dataResults',
};
export const initialState = {
view: VIEW.camera,
};
const stateSlice = createSlice({
name: 'state',
initialState,
reducers: {
setView(state, { payload }) {
state.view = payload;
},
},
});
export const {
setView,
} = stateSlice.actions;
export default stateSlice.reducer;

View File

@@ -46,7 +46,7 @@ module.exports = {
contentBase: commonPaths.outputPath, contentBase: commonPaths.outputPath,
compress: true, compress: true,
hot: true, hot: true,
port: 3000, port: 3333,
}, },
plugins: [ plugins: [
new HotModuleReplacementPlugin(), new HotModuleReplacementPlugin(),

9587
frontend/yarn.lock Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -3,6 +3,7 @@
"version": "1.0.0", "version": "1.0.0",
"scripts": { "scripts": {
"heroku": "git subtree push --prefix api heroku master", "heroku": "git subtree push --prefix api heroku master",
"predeploy": "cd frontend && npm run bundle",
"deploy": "node -r dotenv/config deploy/ftp.js" "deploy": "node -r dotenv/config deploy/ftp.js"
}, },
"devDependencies": { "devDependencies": {