zaphyra's git: oeffisearch

implement own request functions for `vendo-client` and `hafas-client`

To reduce amout of shimming-code and by that reduce bundle-size and
external dependencies.

diff --git a/nginx.conf b/nginx.conf
@@ -34,18 +34,18 @@ http {
 
 	map $hafastarget $hafasurl {
 		default no;
-		nahsh   nah.sh.hafas.de;
-		rmv     www.rmv.de;
-		bvg     bvg-apps-ext.hafas.de;
-		oebb    fahrplan.oebb.at;
+		NAHSH   nah.sh.hafas.de;
+		RMV     www.rmv.de;
+		VBB     bvg-apps-ext.hafas.de;
+		OEBB    fahrplan.oebb.at;
 	}
 
 	map $hafastarget $hafaspath {
 		default no;
-		nahsh   '/bin/mgate.exe';
-		rmv     '/auskunft/bin/jp/mgate.exe';
-		bvg     '/bin/mgate.exe';
-		oebb    '/bin/mgate.exe';
+		NAHSH   '/bin/mgate.exe';
+		RMV     '/auskunft/bin/jp/mgate.exe';
+		VBB     '/bin/mgate.exe';
+		OEBB    '/bin/mgate.exe';
 	}
 
 	server {

diff --git a/package.json b/package.json
@@ -12,7 +12,6 @@
   "author": "Katja(ctucx), yuka",
   "license": "AGPL-3.0",
   "dependencies": {
-    "buffer": "^6.0.3",
     "db-vendo-client": "^6.10.10",
     "hafas-client": "^6.3.6",
     "ics": "^3.12.0",

@@ -32,7 +31,6 @@
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-delete": "^3.0.2",
     "rollup-plugin-html-literals": "^1.1.8",
-    "rollup-plugin-ignore": "^1.0.10",
     "rollup-plugin-lit-css": "^5.0.2",
     "rollup-plugin-summary": "^3.0.1",
     "rollup-plugin-workbox": "^8.1.3"

diff --git a/rollup.config.js b/rollup.config.js
@@ -2,7 +2,6 @@ import path from 'path';
 import del from 'rollup-plugin-delete';
 import copy from 'rollup-plugin-copy';
 import { fileURLToPath } from 'url';
-import ignore from "rollup-plugin-ignore"
 import replace from '@rollup/plugin-replace';
 import resolve from '@rollup/plugin-node-resolve';
 import terser from '@rollup/plugin-terser';

@@ -26,6 +25,18 @@ const gitVersion    = process.env.GIT_VERSION;
 const gitCommit     = process.env.GIT_COMMIT;
 const gitCommitDate = process.env.GIT_COMMITDATE
 
+const ignore = (list, options) => ({
+	resolveId(importee) { return (
+		(
+			importee === 'empty_shim'
+			|| list.includes(importee)
+		)
+		? 'empty_shim'
+		: null
+	); },
+	load(id) { return id === 'empty_shim' ? 'export default {}' : null; }
+});
+
 export default {
 	input: 'src/main.js',
 	output: {

@@ -60,27 +71,18 @@ export default {
 		// Minify HTML template literals
 		!isDevServer && minifyHtmlLiterals(),
 		// stub some modules
-		ignore([ 'http', 'url', 'tls', 'stream', 'assert', 'https-proxy-agent', 'db-hafas-stations', 'events' ]),
+		ignore([ 'db-hafas-stations' ]),
 		replace({
 			preventAssignment: true,
 			delimiters: ['', ''],
 			values: {
-				'import {createRequire} from \'module\';': '',
-	 			'const require = createRequire(import.meta.url);': '',
-				'console.log(url);': '',
-				'process.browser': JSON.stringify(),
-				'process.version': JSON.stringify("0.0"),
+				'import {request} from \'../lib/request.js\';': 'const request = null;',
 				'process.env.DEBUG':         JSON.stringify(),
-				'process.env.HTTPS_PROXY':   JSON.stringify(),
-				'process.env.HTTP_PROXY':    JSON.stringify(),
-				'process.env.LOCAL_ADDRESS': JSON.stringify(),
 			},
 		}),
 		replace({
 			preventAssignment: true,
 			values: {
-				'node:buffer':       'buffer',
-				'cross-fetch':       path.resolve(__dirname, 'src/shim/cross-fetch.js'),
 				'isDevServer':       isDevServer,
 				'APP_NAME':          JSON.stringify(appName),
 				'APP_REPOURL':       JSON.stringify(appRepoUrl),

@@ -97,11 +99,7 @@ export default {
 		// json import support
 		jsonImport(),
 		// Resolve bare module specifiers to relative paths
-		resolve({
-			browser: true,
-			preferBuiltins: false,
-			modulePaths: [ path.resolve(__dirname, 'src/shim') ],
-		}),
+		resolve({ browser: true, preferBuiltins: true }),
 		// Minify JS
 		!isDevServer && terser({
 			ecma: 2021,

diff --git a/src/hafasClient.js b/src/hafasClient.js
@@ -8,11 +8,23 @@ import { profile as rmvProfile         } from 'hafas-client/p/rmv/index.js';
 import { profile as oebbProfile        } from 'hafas-client/p/oebb/index.js';
 import { profile as rejseplanenProfile } from 'hafas-client/p/rejseplanen/index.js'
 
+import { vendoRequest } from './vendoRequest.js';
+import { hafasRequest } from './hafasRequest.js';
+
 const clients  = {};
 
+const vendoEndpoints = {
+	locationsEndpoint: '/db/vendo/locations',
+	stopEndpoint: '/db/vendo/location',
+	journeysEndpoint: '/db/vendo/journeys',
+	refreshJourneysEndpointTickets: '/db/vendo/journey',
+	boardEndpoint: '/db/vendo/departures',
+	tripEndpoint: '/db/vendo/trip',
+};
+
 export let   client;
 export const profiles = {
-	db:          { name: "DB",          backend: 'vendo', profile: dbNavProfile },
+	db:          { name: "DB",          backend: 'vendo', profile: { ...dbNavProfile, ...vendoEndpoints }},
 	bvg:         { name: "BVG",         backend: 'hafas', profile: bvgProfile },
 	nahsh:       { name: "NAH.SH",      backend: 'hafas', profile: nahshProfile },
 	rmv:         { name: "RMV",         backend: 'hafas', profile: rmvProfile },

@@ -26,12 +38,12 @@ export const getHafasClient = async profileName => {
 
 	if (!clients[profileName]) {
 		if (profiles[profileName].backend === 'vendo') {
-			clients[profileName] = createVendoClient(profiles[profileName].profile, APP_NAME, {enrichStations: false});
+			clients[profileName] = createVendoClient({ ...profiles[profileName].profile, request: vendoRequest }, APP_NAME, {enrichStations: false});
 			if (isDevServer) console.info('initialized vendo client with profile ' + profileName);
 		}
 
 		if (profiles[profileName].backend === 'hafas') {
-			clients[profileName] = createHafasClient(profiles[profileName].profile, APP_NAME);
+			clients[profileName] = createHafasClient({ ...profiles[profileName].profile, request: hafasRequest }, APP_NAME);
 			if (isDevServer) console.info('initialized hafas client with profile ' + profileName);
 		}
 	}

diff --git a/src/hafasRequest.js b/src/hafasRequest.js
@@ -0,0 +1,118 @@
+import { HafasError, byErrorCode } from 'hafas-client/lib/errors.js';
+
+const
+	randomBytesHexString = length => [...Array(length)].map(
+		() => Math.floor(Math.random() * 16).toString(16)
+	).join('')
+
+export const
+	checkIfHafasResponseIsOk = (_) => {
+		const {
+			body,
+			errProps: baseErrProps,
+		} = _;
+
+		const errProps = { ...baseErrProps };
+
+		if (body.id) errProps.hafasResponseId = body.id;
+
+		// Because we want more accurate stack traces, we don't construct the error here,
+		// but only return the constructor & error message.
+		const getError = (_) => {
+			// mutating here is ugly but pragmatic
+			if (_.errTxt) errProps.hafasMessage = _.errTxt;
+			if (_.errTxtOut) errProps.hafasDescription = _.errTxtOut;
+			if (_.err in byErrorCode) return byErrorCode[_.err];
+
+			return {
+				Error: HafasError,
+				message: body.errTxt || 'unknown error',
+				props: {},
+			};
+		};
+
+		if (body.err && body.err !== 'OK') {
+			const { Error: HafasError, message, props } = getError(body);
+			throw new HafasError(message, body.err, { ...errProps, ...props });
+		}
+
+		if (!body.svcResL || !body.svcResL[0])
+			throw new HafasError('invalid/unsupported response structure', null, errProps);
+
+		if (body.svcResL[0].err !== 'OK') {
+			const {Error: HafasError, message, props} = getError(body.svcResL[0]);
+			throw new HafasError(message, body.svcResL[0].err, { ...errProps, ...props });
+		}
+	},
+
+	hafasRequest = async (ctx, userAgent, reqData) => {
+		const {profile, opt} = ctx;
+
+		if (profile.addChecksum)
+			throw new Error('profile.addChecksum set but not implemented!');
+		if (profile.addMicMac)
+			throw new Error('profile.addMicMac set but not implemented!');
+
+		const
+			rawReqBody = profile.transformReqBody(ctx, {
+				// todo: is it `eng` actually?
+				// RSAG has `deu` instead of `de`
+				lang: opt.language || profile.defaultLanguage || 'en',
+				svcReqL: [reqData],
+
+				client: profile.client, // client identification
+				ext: profile.ext, // ?
+				ver: profile.ver, // HAFAS protocol version
+				auth: profile.auth, // static authentication
+			}),
+
+			reqId = randomBytesHexString(3).toString('hex'),
+			req = profile.transformReq(ctx, {
+				agent: null,
+				method: 'post',
+				// todo: CORS? referrer policy?
+				body: JSON.stringify(rawReqBody),
+				headers: {
+					'Content-Type': 'application/json',
+					'Accept-Encoding': 'gzip, br, deflate',
+					'Accept': 'application/json',
+					'user-agent': userAgent,
+					'connection': 'keep-alive', // prevent excessive re-connecting
+				},
+				redirect: 'follow',
+				query: {},
+			}),
+
+			url = `/hafas/${profile.client.id}?${(new URLSearchParams(req.query)).toString()}`,
+			fetchReq = new Request(url, req),
+			res = await fetch(url, req),
+			errProps = {
+				// todo [breaking]: assign as non-enumerable property
+				request: fetchReq,
+				// todo [breaking]: assign as non-enumerable property
+				response: res,
+				url,
+			};
+
+		if (!res.ok) {
+			// todo [breaking]: make this a FetchError or a HafasClientError?
+			const err = new Error(res.statusText);
+			Object.assign(err, errProps);
+			throw err;
+		}
+
+		let cType = res.headers.get('content-type');
+		if (!cType.startsWith('application/json'))
+			throw new HafasError('invalid/unsupported response content-type: ' + cType, null, errProps);
+
+		const body = JSON.parse(await res.text())
+
+		checkIfHafasResponseIsOk({ body, errProps });
+
+		const svcRes = body.svcResL[0].res;
+
+		return {
+			res: svcRes,
+			common: profile.parseCommon({...ctx, res: svcRes}),
+		};
+	};

diff --git a/src/shim/cross-fetch.js b/src/shim/cross-fetch.js
@@ -1,28 +0,0 @@
-export const Request = globalThis.Request;
-
-export const fetch = (resource, options) => {
-	if (!(resource instanceof Request)) resource = new Request(resource);
-
-	if (isDevServer) console.log('fetch(): ', resource, options);
-
-	const replacement = {
-		'https://app.services-bahn.de/mob/location/search':        '/db/vendo/locations',
-		'https://app.services-bahn.de/mob/angebote/fahrplan':      '/db/vendo/journeys',
-		'https://app.services-bahn.de/mob/angebote/recon':         '/db/vendo/journey',
-		'https://app.services-bahn.de/mob/bahnhofstafel/abfahrt':  '/db/vendo/departures',
-		'https://nah.sh.hafas.de/bin/mgate.exe?':                  '/hafas/nahsh',
-		'https://www.rmv.de/auskunft/bin/jp/mgate.exe?':           '/hafas/rmv',
-		'https://bvg-apps-ext.hafas.de/bin/mgate.exe?':            '/hafas/bvg',
-		'https://fahrplan.oebb.at/bin/mgate.exe?':                 '/hafas/oebb',
-	};
-
-	let url = replacement[resource.url];
-
-	if (url === undefined) {
-		url = resource.url.replace('https://app.services-bahn.de/mob/location/details', '/db/vendo/location');
-		url = url.replace('https://app.services-bahn.de/mob/zuglauf',                   '/db/vendo/trip');
-	}
-
-	resource = new Request(url, resource);
-	return globalThis.fetch(resource, options);
-}

diff --git a/src/shim/crypto.js b/src/shim/crypto.js
@@ -1,8 +0,0 @@
-import { Buffer } from "buffer";
-
-export const randomBytes = (len, cb) => {
-	if (cb) throw "async not supported";
-	const arr = new Uint8Array(len);
-	self.crypto.getRandomValues(arr);
-	return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
-};

diff --git a/src/shim/https-proxy-agent.js b/src/shim/https-proxy-agent.js
@@ -1,8 +0,0 @@
-import {Agent} from "https";
-
-export default class HttpsProxyAgent extends Agent {
-	constructor(proxy, opts) {
-		super(opts);
-		this.proxy = proxy;
-	}
-}

diff --git a/src/shim/https.js b/src/shim/https.js
@@ -1,3 +0,0 @@
-export class Agent {
-	Agent() {}
-}

diff --git a/src/shim/net.js b/src/shim/net.js
@@ -1 +0,0 @@
-export const isIP = () => {};

diff --git a/src/vendoRequest.js b/src/vendoRequest.js
@@ -0,0 +1,96 @@
+import { parse as parseContentType } from 'content-type';
+import { HafasError } from 'db-vendo-client/lib/errors.js';
+
+export const
+	checkIfVendoResponseIsOk = (_) => {
+		const
+			{ body, errProps: baseErrProps } = _,
+			errProps = { ...baseErrProps };
+
+		if (body.id)
+			errProps.hafasResponseId = body.id;
+
+		// Because we want more accurate stack traces, we don't construct the error here,
+		// but only return the constructor & error message.
+		const getError = (_) => {
+			// mutating here is ugly but pragmatic
+			if (_.fehlerNachricht.ueberschrift) errProps.hafasMessage = _.fehlerNachricht.ueberschrift;
+			if (_.fehlerNachricht.text) errProps.hafasDescription = _.fehlerNachricht.text;
+
+			return {
+				Error: HafasError,
+				message: errProps.hafasMessage || 'unknown error',
+				props: { code: _.fehlerNachricht.code },
+			};
+		};
+
+		if (body.fehlerNachricht || body.errors) { // TODO better handling
+			const {Error: HafasError, message, props} = getError(body);
+			throw new HafasError(message, body.err || body.errors, {...errProps, ...props});
+		}
+	},
+
+	vendoRequest = async (ctx, userAgent, reqData) => {
+		const
+			{profile, opt} = ctx,
+			endpoint = reqData.endpoint;
+
+		delete reqData.endpoint;
+
+		const
+			rawReqBody = profile.transformReqBody(ctx, reqData.body),
+			reqOptions = profile.transformReq(ctx, {
+				agent: null,
+				keepalive: true,
+				method: reqData.method,
+				redirect: 'follow',
+				body: JSON.stringify(rawReqBody),
+				query: reqData.query,
+				headers: {
+					'Content-Type': 'application/json',
+					// 'Accept-Encoding': 'gzip, deflate, br, zstd',
+					'Accept': 'application/json',
+					'Accept-Language': opt.language || profile.defaultLanguage || 'en',
+					'user-agent': userAgent,
+					...reqData.headers,
+				},
+			}),
+
+			url = `${[ endpoint, reqData?.path ].filter(x => typeof x === 'string' && x.length > 0).join('/')}?${(new URLSearchParams(reqOptions.query)).toString()}`,
+			fetchReq = new Request(url, reqOptions),
+			res = await fetch(url, reqOptions),
+			errProps = {
+				// todo [breaking]: assign as non-enumerable property
+				request: fetchReq,
+				// todo [breaking]: assign as non-enumerable property
+				response: res,
+				url,
+			};
+
+		if (!res.ok) {
+			// todo [breaking]: make this a FetchError or a HafasClientError?
+			console.log(JSON.stringify(res), await res.text());
+			const err = new Error(res.statusText);
+			Object.assign(err, errProps);
+			throw err;
+		}
+
+		const cType = res.headers.get('content-type');
+		if (cType) {
+			const { type } = parseContentType(cType);
+			// For some reason, the reqOptions.headers object is sometimes a plain object
+			// and sometimes a Headers object (In browser env). For the latter, .get() must
+			// be used.
+			if (type !== reqOptions.headers['Accept'] && type !== reqOptions.headers.get('Accept'))
+				throw new HafasError('invalid/unsupported response content-type: ' + cType, null, errProps);
+		}
+
+		const body = JSON.parse(await res.text());
+
+		checkIfVendoResponseIsOk({ body, errProps });
+
+		return {
+			res: body,
+			common: {},
+		};
+	};