From 9f4f05300e3c3cea2f0ae9b819233e1e46788c5d Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:24:01 +0300 Subject: [PATCH 1/9] feat: add dataset id enum, operators, and metadata transform --- search_dataset_schema.js | 34 ++++++++++++++++++++ test/search-dataset-schema.test.js | 51 ++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 search_dataset_schema.js create mode 100644 test/search-dataset-schema.test.js diff --git a/search_dataset_schema.js b/search_dataset_schema.js new file mode 100644 index 0000000..19a40e9 --- /dev/null +++ b/search_dataset_schema.js @@ -0,0 +1,34 @@ +'use strict'; /*jslint node:true es9:true*/ +import {z} from 'zod'; + +export const DATASET_IDS = [ + 'gd_l1viktl72bvl7bjuj0', + 'gd_me5ppxjr2ge6icjuh0', + 'gd_l1vikfnt1wgvvqz95w', +]; + +export const dataset_id_schema = z.enum(DATASET_IDS); + +export const FILTER_OPERATORS = ['=', '!=', '<', '<=', '>', '>=', 'in', + 'not_in', 'includes', 'not_includes', 'array_includes', + 'not_array_includes', 'is_null', 'is_not_null']; + +export function metadata_to_fields(metadata){ + const fields = metadata && typeof metadata=='object' + && metadata.fields && typeof metadata.fields=='object' + ? metadata.fields : {}; + const out = []; + for (const [name, meta] of Object.entries(fields)) + { + if (!meta || typeof meta!='object') + continue; + if (meta.active===false) + continue; + out.push({ + name, + type: meta.type, + description: meta.description, + }); + } + return out; +} diff --git a/test/search-dataset-schema.test.js b/test/search-dataset-schema.test.js new file mode 100644 index 0000000..00c2e60 --- /dev/null +++ b/test/search-dataset-schema.test.js @@ -0,0 +1,51 @@ +'use strict'; /*jslint node:true es9:true*/ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {DATASET_IDS, dataset_id_schema, metadata_to_fields, FILTER_OPERATORS} + from '../search_dataset_schema.js'; + +test('DATASET_IDS lists the three supported datasets', ()=>{ + assert.deepEqual(DATASET_IDS, [ + 'gd_l1viktl72bvl7bjuj0', + 'gd_me5ppxjr2ge6icjuh0', + 'gd_l1vikfnt1wgvvqz95w', + ]); +}); + +test('dataset_id_schema accepts a supported id', ()=>{ + assert.equal(dataset_id_schema.parse('gd_l1viktl72bvl7bjuj0'), + 'gd_l1viktl72bvl7bjuj0'); +}); + +test('dataset_id_schema rejects an unknown id', ()=>{ + assert.throws(()=>dataset_id_schema.parse('gd_not_a_real_dataset')); +}); + +test('FILTER_OPERATORS lists the documented operators', ()=>{ + assert.ok(FILTER_OPERATORS.includes('includes')); + assert.ok(FILTER_OPERATORS.includes('is_null')); + assert.ok(FILTER_OPERATORS.includes('not_array_includes')); +}); + +test('metadata_to_fields keeps active fields as name/type/description', ()=>{ + const metadata = { + id: 'gd_l1vijqt9jfj7olije', + fields: { + name: {type: 'text', active: true, + description: 'The name of the company'}, + url: {type: 'url', required: true, + description: 'The company URL'}, + cb_rank: {type: 'number', active: false, + description: 'Crunchbase rank'}, + }, + }; + assert.deepEqual(metadata_to_fields(metadata), [ + {name: 'name', type: 'text', description: 'The name of the company'}, + {name: 'url', type: 'url', description: 'The company URL'}, + ]); +}); + +test('metadata_to_fields tolerates missing fields object', ()=>{ + assert.deepEqual(metadata_to_fields({id: 'x'}), []); + assert.deepEqual(metadata_to_fields(null), []); +}); From bb50204bbf18b3b49a742494d31879fb10ad153d Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:26:16 +0300 Subject: [PATCH 2/9] test: tighten operator and metadata transform coverage --- test/search-dataset-schema.test.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/search-dataset-schema.test.js b/test/search-dataset-schema.test.js index 00c2e60..b67dba0 100644 --- a/test/search-dataset-schema.test.js +++ b/test/search-dataset-schema.test.js @@ -22,9 +22,13 @@ test('dataset_id_schema rejects an unknown id', ()=>{ }); test('FILTER_OPERATORS lists the documented operators', ()=>{ - assert.ok(FILTER_OPERATORS.includes('includes')); - assert.ok(FILTER_OPERATORS.includes('is_null')); - assert.ok(FILTER_OPERATORS.includes('not_array_includes')); + assert.deepEqual(FILTER_OPERATORS, [ + '=', '!=', '<', '<=', '>', '>=', + 'in', 'not_in', + 'includes', 'not_includes', + 'array_includes', 'not_array_includes', + 'is_null', 'is_not_null', + ]); }); test('metadata_to_fields keeps active fields as name/type/description', ()=>{ @@ -49,3 +53,9 @@ test('metadata_to_fields tolerates missing fields object', ()=>{ assert.deepEqual(metadata_to_fields({id: 'x'}), []); assert.deepEqual(metadata_to_fields(null), []); }); + +test('metadata_to_fields skips non-object field entries', ()=>{ + assert.deepEqual(metadata_to_fields({fields: {bad: null, + ok: {type: 'text', description: 'fine'}}}), + [{name: 'ok', type: 'text', description: 'fine'}]); +}); From 0eff4d21e63489818a33bb1c5a77f9ffabab5f7d Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:27:50 +0300 Subject: [PATCH 3/9] feat: add depth-capped recursive filter schema --- search_dataset_schema.js | 29 ++++++++++++++++++ test/search-dataset-schema.test.js | 49 ++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/search_dataset_schema.js b/search_dataset_schema.js index 19a40e9..06b87cb 100644 --- a/search_dataset_schema.js +++ b/search_dataset_schema.js @@ -13,6 +13,35 @@ export const FILTER_OPERATORS = ['=', '!=', '<', '<=', '>', '>=', 'in', 'not_in', 'includes', 'not_includes', 'array_includes', 'not_array_includes', 'is_null', 'is_not_null']; +const leaf_value_schema = z.union([ + z.string(), + z.number(), + z.boolean(), + z.array(z.union([z.string(), z.number(), z.boolean()])), +]); + +const leaf_schema = z.object({ + name: z.string().describe('Field name to filter on. Get valid field ' + +'names from the list_dataset_fields tool.'), + operator: z.string().describe('Filter operator, one of: ' + +FILTER_OPERATORS.join(', ')), + value: leaf_value_schema, +}); + +// Build the filter node schema with a hard nesting cap (max depth 3). +// At max depth, only leaf nodes are allowed. +function build_node_schema(depth){ + if (depth<=1) + return leaf_schema; + const group_schema = z.object({ + operator: z.enum(['and', 'or']), + filters: z.array(build_node_schema(depth-1)).min(1), + }); + return z.union([group_schema, leaf_schema]); +} + +export const filter_schema = build_node_schema(4); + export function metadata_to_fields(metadata){ const fields = metadata && typeof metadata=='object' && metadata.fields && typeof metadata.fields=='object' diff --git a/test/search-dataset-schema.test.js b/test/search-dataset-schema.test.js index b67dba0..5dc8a66 100644 --- a/test/search-dataset-schema.test.js +++ b/test/search-dataset-schema.test.js @@ -3,6 +3,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import {DATASET_IDS, dataset_id_schema, metadata_to_fields, FILTER_OPERATORS} from '../search_dataset_schema.js'; +import {filter_schema} from '../search_dataset_schema.js'; test('DATASET_IDS lists the three supported datasets', ()=>{ assert.deepEqual(DATASET_IDS, [ @@ -59,3 +60,51 @@ test('metadata_to_fields skips non-object field entries', ()=>{ ok: {type: 'text', description: 'fine'}}}), [{name: 'ok', type: 'text', description: 'fine'}]); }); + +test('filter_schema accepts the documented flat example', ()=>{ + const filter = {operator: 'and', filters: [ + {name: 'name', value: 'Egor', operator: 'includes'}, + ]}; + assert.deepEqual(filter_schema.parse(filter), filter); +}); + +test('filter_schema accepts a single leaf node', ()=>{ + const filter = {name: 'cb_rank', value: 100, operator: '<'}; + assert.deepEqual(filter_schema.parse(filter), filter); +}); + +test('filter_schema accepts array and boolean leaf values', ()=>{ + const filter = {operator: 'or', filters: [ + {name: 'tags', value: ['a', 'b'], operator: 'array_includes'}, + {name: 'verified', value: true, operator: '='}, + ]}; + assert.deepEqual(filter_schema.parse(filter), filter); +}); + +test('filter_schema accepts nesting up to depth 3', ()=>{ + const filter = {operator: 'and', filters: [ + {operator: 'or', filters: [ + {operator: 'and', filters: [ + {name: 'name', value: 'x', operator: 'includes'}, + ]}, + ]}, + ]}; + assert.deepEqual(filter_schema.parse(filter), filter); +}); + +test('filter_schema rejects nesting deeper than 3', ()=>{ + const filter = {operator: 'and', filters: [ + {operator: 'or', filters: [ + {operator: 'and', filters: [ + {operator: 'or', filters: [ + {name: 'name', value: 'x', operator: 'includes'}, + ]}, + ]}, + ]}, + ]}; + assert.throws(()=>filter_schema.parse(filter)); +}); + +test('filter_schema rejects an empty object', ()=>{ + assert.throws(()=>filter_schema.parse({})); +}); From 315b1294a31a394eeeeb0e005bdee59d33199a5d Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:30:17 +0300 Subject: [PATCH 4/9] refactor: name the filter nesting cap and merge test imports --- search_dataset_schema.js | 5 ++++- test/search-dataset-schema.test.js | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/search_dataset_schema.js b/search_dataset_schema.js index 06b87cb..a195d9f 100644 --- a/search_dataset_schema.js +++ b/search_dataset_schema.js @@ -40,7 +40,10 @@ function build_node_schema(depth){ return z.union([group_schema, leaf_schema]); } -export const filter_schema = build_node_schema(4); +// depth param counts schema layers incl. the leaf layer, so the value is +// MAX_NESTING + 1 to allow exactly MAX_NESTING levels of group nesting. +const MAX_NESTING = 3; +export const filter_schema = build_node_schema(MAX_NESTING+1); export function metadata_to_fields(metadata){ const fields = metadata && typeof metadata=='object' diff --git a/test/search-dataset-schema.test.js b/test/search-dataset-schema.test.js index 5dc8a66..304787a 100644 --- a/test/search-dataset-schema.test.js +++ b/test/search-dataset-schema.test.js @@ -1,9 +1,8 @@ 'use strict'; /*jslint node:true es9:true*/ import test from 'node:test'; import assert from 'node:assert/strict'; -import {DATASET_IDS, dataset_id_schema, metadata_to_fields, FILTER_OPERATORS} - from '../search_dataset_schema.js'; -import {filter_schema} from '../search_dataset_schema.js'; +import {DATASET_IDS, dataset_id_schema, metadata_to_fields, FILTER_OPERATORS, + filter_schema} from '../search_dataset_schema.js'; test('DATASET_IDS lists the three supported datasets', ()=>{ assert.deepEqual(DATASET_IDS, [ From 7286d3232cd517272b9713cc548ccacdbe9b05e9 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:31:36 +0300 Subject: [PATCH 5/9] feat: add list_dataset_fields tool --- server.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/server.js b/server.js index ff9e086..ee85757 100644 --- a/server.js +++ b/server.js @@ -7,6 +7,8 @@ import {tools as browser_tools} from './browser_tools.js'; import prompts from './prompts.js'; import {GROUPS} from './tool_groups.js'; import {parse_google_search_response} from './search_utils.js'; +import {dataset_id_schema, metadata_to_fields} + from './search_dataset_schema.js'; import {createRequire} from 'node:module'; import {remark} from 'remark'; import strip from 'strip-markdown'; @@ -603,6 +605,36 @@ addTool({ }), }); +const SEARCHABLE_DATASETS_DESC = [ + 'Supported dataset_id values:', + '- gd_l1viktl72bvl7bjuj0: LinkedIn people profiles', + '- gd_me5ppxjr2ge6icjuh0: LinkedIn people profiles (contact-enriched)', + '- gd_l1vikfnt1wgvvqz95w: LinkedIn company information', +].join('\n'); + +addTool({ + name: 'list_dataset_fields', + description: 'List the filterable fields of a searchable dataset ' + +'(field name, type, and description). Call this before ' + +'search_dataset to learn which field names and types you can ' + +'filter on.\n'+SEARCHABLE_DATASETS_DESC, + annotations: { + title: 'List Dataset Fields', + readOnlyHint: true, + openWorldHint: true, + }, + parameters: z.object({dataset_id: dataset_id_schema}), + execute: tool_fn('list_dataset_fields', async({dataset_id}, ctx)=>{ + let response = await base_request({ + url: `https://api.brightdata.com/datasets/${dataset_id}` + +`/metadata`, + method: 'GET', + headers: api_headers(ctx.clientName, 'list_dataset_fields'), + }); + return JSON.stringify(metadata_to_fields(response.data)); + }), +}); + addTool({ name: 'session_stats', description: 'Tell the user about the tool usage during this session', From b06e5aa1f59087fef26eb3d743889f87d9089ffe Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:35:56 +0300 Subject: [PATCH 6/9] feat: add search_dataset filter-search tool --- server.js | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index ee85757..ba6cf90 100644 --- a/server.js +++ b/server.js @@ -7,7 +7,7 @@ import {tools as browser_tools} from './browser_tools.js'; import prompts from './prompts.js'; import {GROUPS} from './tool_groups.js'; import {parse_google_search_response} from './search_utils.js'; -import {dataset_id_schema, metadata_to_fields} +import {dataset_id_schema, filter_schema, metadata_to_fields, FILTER_OPERATORS} from './search_dataset_schema.js'; import {createRequire} from 'node:module'; import {remark} from 'remark'; @@ -635,6 +635,63 @@ addTool({ }), }); +addTool({ + name: 'search_dataset', + description: 'Search a Bright Data dataset by a filter and get matching ' + +'records back directly (fast Elasticsearch-backed search; no ' + +'snapshot). Use this to FIND MANY records by criteria, as opposed ' + +'to the web_data_* tools which fetch ONE record by URL.\n' + +'First call list_dataset_fields to get valid field names.\n' + +'A filter is a tree: a group {operator:"and"|"or", filters:[...]} ' + +'or a leaf {name, value, operator}. Max nesting depth 3.\n' + +'Leaf operators: '+FILTER_OPERATORS.join(', ')+'.\n' + +SEARCHABLE_DATASETS_DESC, + annotations: { + title: 'Search Dataset', + readOnlyHint: true, + openWorldHint: true, + }, + parameters: z.object({ + dataset_id: dataset_id_schema, + filter: filter_schema.describe('Filter tree describing which ' + +'records to match. Required, cannot be empty.'), + size: z.number().int().positive().optional().default(100) + .describe('Max number of records to return (default 100)'), + sort: z.union([ + z.enum(['default', 'random']), + z.array(z.record(z.enum(['asc', 'desc']))), + ]).optional().describe('Sorting: "default", "random", or a custom ' + +'array like [{"timestamp":"asc"}]. Use "default" or custom ' + +'sort to paginate with search_after.'), + search_after: z.array(z.any()).optional().describe('Pagination ' + +'cursor from a previous response\'s search_after value.'), + }), + execute: tool_fn('search_dataset', async({dataset_id, filter, size, sort, + search_after}, ctx)=> + { + let body = {mode: 'sync', filter, size}; + if (sort!==undefined) + body.sort = sort; + if (search_after!==undefined) + body.search_after = search_after; + let response = await base_request({ + url: `https://api.brightdata.com/datasets/search/${dataset_id}`, + method: 'POST', + data: body, + headers: { + ...api_headers(ctx.clientName, 'search_dataset'), + 'Content-Type': 'application/json', + }, + }); + let {hits, total_hits, took, search_after: next_cursor} + = response.data || {}; + let result = {hits, total_hits, took}; + if (next_cursor!==undefined) + result.search_after = next_cursor; + return JSON.stringify(result); + }), +}); + addTool({ name: 'session_stats', description: 'Tell the user about the tool usage during this session', From d4832ca3c6eb787fe89d8b8176790d945d93dcd0 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:38:52 +0300 Subject: [PATCH 7/9] chore: register dataset search tools in groups and manifest --- package.json | 1 + tool_groups.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/package.json b/package.json index 1e96cfc..4023e29 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "files": [ "server.js", "search_utils.js", + "search_dataset_schema.js", "browser_tools.js", "browser_session.js", "aria_snapshot_filter.js", diff --git a/tool_groups.js b/tool_groups.js index a9027fd..c21d75d 100644 --- a/tool_groups.js +++ b/tool_groups.js @@ -33,6 +33,8 @@ export const GROUPS = { 'web_data_linkedin_job_listings', 'web_data_linkedin_posts', 'web_data_linkedin_people_search', + 'list_dataset_fields', + 'search_dataset', 'web_data_instagram_profiles', 'web_data_instagram_posts', 'web_data_instagram_reels', @@ -95,6 +97,8 @@ export const GROUPS = { 'web_data_google_maps_reviews', 'web_data_zillow_properties_listing', 'web_data_booking_hotel_listings', + 'list_dataset_fields', + 'search_dataset', ], }, RESEARCH: { From 754ff8d9deb7b2b49854d11260c52e55543724fe Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 13:56:37 +0300 Subject: [PATCH 8/9] feat: cap search_dataset size at max 10 (default 10) --- server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index ba6cf90..4bd8b34 100644 --- a/server.js +++ b/server.js @@ -655,8 +655,8 @@ addTool({ dataset_id: dataset_id_schema, filter: filter_schema.describe('Filter tree describing which ' +'records to match. Required, cannot be empty.'), - size: z.number().int().positive().optional().default(100) - .describe('Max number of records to return (default 100)'), + size: z.number().int().positive().max(10).optional().default(10) + .describe('Number of records to return (max 10, default 10)'), sort: z.union([ z.enum(['default', 'random']), z.array(z.record(z.enum(['asc', 'desc']))), From 3dd879747857f972a8145811d621938db61ceece Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Mon, 1 Jun 2026 14:04:09 +0300 Subject: [PATCH 9/9] style: remove comments from search_dataset_schema.js --- search_dataset_schema.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/search_dataset_schema.js b/search_dataset_schema.js index a195d9f..305e03a 100644 --- a/search_dataset_schema.js +++ b/search_dataset_schema.js @@ -28,8 +28,6 @@ const leaf_schema = z.object({ value: leaf_value_schema, }); -// Build the filter node schema with a hard nesting cap (max depth 3). -// At max depth, only leaf nodes are allowed. function build_node_schema(depth){ if (depth<=1) return leaf_schema; @@ -40,8 +38,6 @@ function build_node_schema(depth){ return z.union([group_schema, leaf_schema]); } -// depth param counts schema layers incl. the leaf layer, so the value is -// MAX_NESTING + 1 to allow exactly MAX_NESTING levels of group nesting. const MAX_NESTING = 3; export const filter_schema = build_node_schema(MAX_NESTING+1);