Remove data masking

modules/server/.env.schema

@@ -1,10 +1,8 @@
 ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS=false
 CONFIG_PATH=./configs
-DATA_MASK_MIN_THRESHOLD=
 DEBUG=false
 DOCUMENT_TYPE=''
 DOWNLOAD_STREAM_BUFFER_SIZE=2000
-ENABLE_DOCUMENT_HITS=true
 ENABLE_LOGS=false
 ES_HOST=http://localhost:9200
 ES_INDEX=''

modules/server/src/config/constants.ts

@@ -2,13 +2,10 @@ import { stringToBool, stringToNumber } from '#utils/stringFns.js';
 
 export const ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS = stringToBool(process.env.ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS);
 export const CONFIG_FILES_PATH = process.env.CONFIG_PATH || './configs';
-export const DATA_MASK_MIN_THRESHOLD = stringToNumber(process.env.DATA_MASK_MIN_THRESHOLD) || Number.MAX_SAFE_INTEGER;
 export const DEBUG_MODE = stringToBool(process.env.DEBUG);
 export const DOCUMENT_TYPE = process.env.DOCUMENT_TYPE || '';
 export const DOWNLOAD_STREAM_BUFFER_SIZE = stringToNumber(process.env.DOWNLOAD_STREAM_BUFFER_SIZE) || 2000;
 export const ENABLE_ADMIN = stringToBool(process.env.ENABLE_ADMIN);
-export const ENABLE_DOCUMENT_HITS =
-	[undefined, ''].includes(process.env.ENABLE_DOCUMENT_HITS) || stringToBool(process.env.ENABLE_DOCUMENT_HITS);
 export const ENABLE_LOGS = stringToBool(process.env.ENABLE_LOGS);
 export const ENABLE_NETWORK_AGGREGATION = stringToBool(process.env.ENABLE_NETWORK_AGGREGATION);
 export const ES_ARRANGER_SET_INDEX = process.env.ES_ARRANGER_SET_INDEX || 'arranger-sets';

modules/server/src/graphqlRoutes.js

@@ -116,14 +116,7 @@ const getTypesWithMappings = async (mapping, configs = {}) => {
 	throw Error('  No configs available at getTypesWithMappings');
 };
 
-const createSchema = async ({
-	enableAdmin,
-	enableDocumentHits,
-	getServerSideFilter,
-	graphqlOptions = {},
-	setsIndex,
-	types,
-}) => {
+const createSchema = async ({ enableAdmin, getServerSideFilter, graphqlOptions = {}, setsIndex, types }) => {
 	const schemaBase = {
 		getServerSideFilter,
 		rootTypes: [],
@@ -135,13 +128,11 @@ const createSchema = async ({
 		...(types && {
 			mockSchema: makeSchema({
 				enableAdmin,
-				enableDocumentHits,
 				mock: true,
 				...schemaBase,
 			}),
 			schema: makeSchema({
 				enableAdmin,
-				enableDocumentHits,
 				middleware: graphqlOptions.middleware || [],
 				...schemaBase,
 			}),
@@ -241,9 +232,7 @@ const createEndpoint = async ({ esClient, graphqlOptions = {}, mockSchema, schem
 export const createSchemasFromConfigs = async ({
 	configsSource = '',
 	enableAdmin,
-	enableDocumentHits,
 	enableNetworkAggregation,
-	dataMaskMinThreshold,
 	esClient,
 	getServerSideFilter,
 	graphqlOptions = {},
@@ -258,7 +247,6 @@ export const createSchemasFromConfigs = async ({
 
 		const { mockSchema, schema } = await createSchema({
 			enableAdmin,
-			enableDocumentHits,
 			getServerSideFilter,
 			graphqlOptions,
 			setsIndex,
@@ -308,27 +296,22 @@ export const createSchemasFromConfigs = async ({
 export default async ({
 	configsSource = '',
 	enableAdmin,
-	enableDocumentHits,
 	enableNetworkAggregation,
-	dataMaskMinThreshold,
 	esClient,
 	getServerSideFilter,
 	graphqlOptions = {},
 	setsIndex,
 }) => {
 	try {
-		const { fieldsFromMapping, mockSchema, networkSchemas, schema, typesWithMappings } =
-			await createSchemasFromConfigs({
-				configsSource,
-				enableAdmin,
-				enableDocumentHits,
-				enableNetworkAggregation,
-				dataMaskMinThreshold,
-				esClient,
-				getServerSideFilter,
-				graphqlOptions,
-				setsIndex,
-			});
+		const { fieldsFromMapping, mockSchema, schema, typesWithMappings } = await createSchemasFromConfigs({
+			configsSource,
+			enableAdmin,
+			enableNetworkAggregation,
+			esClient,
+			getServerSideFilter,
+			graphqlOptions,
+			setsIndex,
+		});
 
 		const graphQLEndpoints = await createEndpoint({
 			esClient,

modules/server/src/mapping/createConnectionResolvers.ts

@@ -3,13 +3,12 @@ import { type IResolvers } from '@graphql-tools/utils';
 import { type GetServerSideFilterFn } from '#utils/getDefaultServerSideFilter.js';
 
 import { createResolvers } from './resolvers.js';
+import resolveHits from './resolveHits.js';
 
 // TODO: tighten these types
 export type CreateConnectionResolversArgs = {
 	createStateResolvers?: boolean;
 	enableAdmin: boolean;
-	enableDocumentHits: boolean;
-	dataMaskMinThreshold: number;
 	getServerSideFilter?: GetServerSideFilterFn;
 	Parallel: any;
 	type: Record<string, any>;
@@ -20,8 +19,6 @@ type CreateConnectionResolversFn = (args: CreateConnectionResolversArgs) => IRes
 const createConnectionResolvers: CreateConnectionResolversFn = ({
 	createStateResolvers = true,
 	enableAdmin,
-	enableDocumentHits,
-	dataMaskMinThreshold,
 	getServerSideFilter,
 	Parallel,
 	type,
@@ -31,15 +28,13 @@ const createConnectionResolvers: CreateConnectionResolversFn = ({
 		type,
 		Parallel,
 		getServerSideFilter,
-		enableDocumentHits,
-		dataMaskMinThreshold,
 	});
 
 	return {
 		[type.name]: {
 			aggregations,
 			configs,
-			hits,
+			hits: resolveHits({ type, Parallel, getServerSideFilter }),
 			// keeping this available for backwards compatibility, but hoping to remove it
 			// TODO: investigate its current usage and need. remove otherwise
 			// Update 2023-02: ENABLE_ADMIN prevents error comes up on facets.

modules/server/src/mapping/createConnectionTypeDefs.js

@@ -1,17 +1,13 @@
 import mappingToAggsType from './mappingToAggsType.js';
 
-const createConnectionType = ({ type, enableDocumentHits }) => {
+const createConnectionType = ({ type }) => {
 	return `type ${type.name}Connection {
     total: Int!
-   ${enableDocumentHits ? `edges: [${type.name}Edge]` : ''}
+   edges: [${type.name}Edge]
   }`;
 };
 
-const createDataMaskingType = ({ enableDocumentHits }) => {
-	return !enableDocumentHits ? `type DataMasking { thresholdValue: Int }` : '';
-};
-
-export default ({ type, fields = '', createStateTypeDefs = true, enableDocumentHits }) => {
+export default ({ type, fields = '', createStateTypeDefs = true }) => {
 	return `
     type ${type.name} {
       aggregations(
@@ -42,9 +38,8 @@ export default ({ type, fields = '', createStateTypeDefs = true, enableDocumentH
       ${mappingToAggsType(type.mapping)}
     }
 
-    ${createDataMaskingType({ enableDocumentHits })}
 
-    ${createConnectionType({ type, enableDocumentHits })}
+    ${createConnectionType({ type })}
 
 
     type ${type.name}Edge {

modules/server/src/mapping/mappingToFields.js

@@ -5,14 +5,13 @@ import mappingToNestedFields from './mappingToNestedFields.js';
 import mappingToObjectTypes from './mappingToObjectTypes.js';
 import mappingToScalarFields from './mappingToScalarFields.js';
 
-const mappingToFields = ({ enableDocumentHits, type, parent }) => {
+const mappingToFields = ({ type, parent }) => {
 	return [
 		mappingToObjectTypes(type.name, type.mapping, parent, type.extendedFields),
 		Object.entries(type.mapping)
 			.filter(([, metadata]) => metadata.type === 'nested')
 			.map(([fieldName, metadata]) =>
 				mappingToFields({
-					enableDocumentHits,
 					parent: [parent, fieldName].filter(Boolean).join('.'),
 					type: {
 						...type,
@@ -29,7 +28,6 @@ const mappingToFields = ({ enableDocumentHits, type, parent }) => {
 				type.customFields,
 			],
 			createStateTypeDefs: 'createState' in type ? type.createState : true,
-			enableDocumentHits,
 		}),
 	].join();
 };

modules/server/src/mapping/mappingToObjectTypes.js

@@ -5,29 +5,33 @@ import mappingToNestedTypes from './mappingToNestedTypes.js';
 import mappingToScalarFields from './mappingToScalarFields.js';
 
 let mappingToObjectTypes = (type, mapping, parent, extendedFields) => {
+	if (!mapping) {
+		throw `Invalid type mapping of ${mapping}`;
+	}
+
 	return Object.entries(mapping)
 		.filter(([, metadata]) => !metadata.type && metadata.properties)
 		.map(
 			([field, metadata]) => `
         ${mappingToObjectTypes(
-					type + capitalize(field),
-					metadata.properties,
-					[parent, field].filter(Boolean).join('.'),
-					extendedFields,
-				)},
+			type + capitalize(field),
+			metadata.properties,
+			[parent, field].filter(Boolean).join('.'),
+			extendedFields,
+		)},
         ${mappingToNestedTypes(
-					type + capitalize(field),
-					metadata.properties,
-					[parent, field].filter(Boolean).join('.'),
-					extendedFields,
-				).join('\n')}
+			type + capitalize(field),
+			metadata.properties,
+			[parent, field].filter(Boolean).join('.'),
+			extendedFields,
+		).join('\n')}
         type ${type + capitalize(field)} {
           ${mappingToNestedFields(
-						type + capitalize(field),
-						metadata.properties,
-						[parent, field].filter(Boolean).join('.'),
-						extendedFields,
-					)}
+				type + capitalize(field),
+				metadata.properties,
+				[parent, field].filter(Boolean).join('.'),
+				extendedFields,
+			)}
           ${mappingToScalarFields(metadata.properties, extendedFields, [parent, field].filter(Boolean).join('.'))}
         }
       `,

modules/server/src/mapping/resolveAggregations.ts

@@ -5,14 +5,12 @@ import { buildQuery, buildAggregations, flattenAggregations } from '#middleware/
 import esSearch from './utils/esSearch.js';
 import { resolveSetsInSqon } from './hackyTemporaryEsSetResolution.js';
 import compileFilter from './utils/compileFilter.js';
-import { Relation } from './masking.js';
 import { type GetServerSideFilterFn } from '#utils/getDefaultServerSideFilter.js';
 import { type Resolver, type Root } from '#gqlServer.js';
 
 export type Bucket = {
 	doc_count: number;
 	key: string;
-	relation: Relation;
 };
 
 export type CommonAggregationProperties = {