@@ -295,84 +295,38 @@ def test_custom_training_table_declares_integer_timestamp_unit(graph, unit,
295295 assert _rows (seeds [stage ], columns ) == _rows (original_seeds [stage ], columns )
296296
297297
298- def _reload (graph , name , pquery ):
299- _ , snapshot , connector , _ = graph
300- return kumo .TrainingTable .from_source_table (
301- connector [name ], TrainingTableSpec (split_col = 'SPLIT' ), pq = pquery ,
302- snapshot = snapshot )
303-
304-
305298def test_manifest_and_export_sidecar (graph , tmp_path ):
306299 from kumo_core .training .export import export_train_table
307300
308301 _ , snapshot , connector , out = graph
309302 query = 'PREDICT ents.LABEL FOR EACH ents.ID'
310303 pquery , table , _ = _export (graph , query , 'lineage' )
311304 manifest = table .manifest
312- assert manifest ['kind' ] == 'training_table'
313305 assert manifest ['id' ] == table .tt_id == _spec (table )['tt_id' ]
314306 assert manifest ['parents' ] == {'snapshot' : snapshot .id }
315307 assert (manifest ['query' ], manifest ['source' ]) == (query , 'pql' )
316- assert manifest ['task_type' ] == 'binary_classification'
317308 assert manifest ['rows' ] == _seeds (table )[1 ]
318309 assert manifest ['split_seed' ] == 0
319- assert manifest ['generation_plan' ]['split' ] == 'inferred'
320310
321311 directory = out / 'lineage'
322- parts = sorted (directory .glob ('*.parquet' ))
323312 baseline = tmp_path / 'baseline'
324- expected = export_train_table (str (table .staging_dir ), str (baseline ))
325- assert [p .name for p in parts ] == [p .split ('/' )[- 1 ] for p in expected ]
326- assert [p .read_bytes () for p in parts ] == [
327- (baseline / p .name ).read_bytes () for p in parts
328- ]
329- assert sorted (p .name for p in directory .iterdir ()) == sorted (
330- [p .name for p in parts ] +
331- ['_KUMO_EXPORT' , '_SUCCESS' , '_kumo_manifest.json' ])
332- sidecar = (directory / '_kumo_manifest.json' ).read_bytes ()
333- assert json .loads (sidecar ) == manifest
334-
335- table .export (TrainingTableOutputConfig (connector , 'lineage' ))
336- assert (directory / '_kumo_manifest.json' ).read_bytes () == sidecar
337-
338-
339- def test_custom_table_records_its_source (graph , recwarn ):
340- _ , snapshot , connector , out = graph
341- pquery , table , _ = _export (graph , 'PREDICT ents.LABEL FOR EACH ents.ID' ,
342- 'reloaded' )
343- (out / 'reloaded' / '_kumo_manifest.json' ).write_text ('{not json' )
344- reloaded = _reload (graph , 'reloaded' , pquery )
313+ export_train_table (str (table .staging_dir ), str (baseline ))
314+ for part in baseline .glob ('*.parquet' ):
315+ assert (directory / part .name ).read_bytes () == part .read_bytes ()
316+ assert json .loads (
317+ (directory / '_kumo_manifest.json' ).read_bytes ()) == manifest
318+
319+ reloaded = kumo .TrainingTable .from_source_table (
320+ connector ['lineage' ], TrainingTableSpec (split_col = 'SPLIT' ), pq = pquery ,
321+ snapshot = snapshot )
345322 manifest = reloaded .manifest
346323 assert manifest ['source' ] == 'custom'
347- assert manifest ['id' ] != table .tt_id
348- assert manifest ['parents' ] == {
349- 'snapshot' : snapshot .id ,
350- 'source_table' : {
351- 'connector' : 'LOCAL_FILE' ,
352- 'location' : str (out / 'reloaded' ),
353- 'version' : connector .source_backend .get_data_version (
354- connector , 'reloaded' ),
355- },
324+ assert manifest ['parents' ]['source_table' ] == {
325+ 'connector' : 'LOCAL_FILE' ,
326+ 'location' : str (directory ),
327+ 'version' : connector .source_backend .get_data_version (
328+ connector , 'lineage' ),
356329 }
357- assert manifest ['parents' ]['source_table' ]['version' ]
358- assert not [w for w in recwarn if 'manifest' in str (w .message )]
359- assert _seeds (reloaded )[1 ] == _seeds (table )[1 ]
360-
361-
362- def test_unrecordable_location_is_null (graph , monkeypatch ):
363- from kumo .engine import _provenance
364-
365- pquery , table , _ = _export (graph , 'PREDICT ents.LABEL FOR EACH ents.ID' ,
366- 'unrecorded' )
367-
368- def fail (* args , ** kwargs ):
369- raise OSError ('listing denied' )
370-
371- monkeypatch .setattr (_provenance , 'source_location' , fail )
372- with pytest .warns (UserWarning , match = 'listing denied' ):
373- reloaded = _reload (graph , 'unrecorded' , pquery )
374- recorded = reloaded .manifest ['parents' ]['source_table' ]
375- assert recorded ['location' ] is None and recorded ['version' ]
376330 assert _seeds (reloaded )[1 ] == _seeds (table )[1 ]
377331
378332
@@ -387,37 +341,27 @@ def fail(*args, **kwargs):
387341
388342 monkeypatch .setattr (training_table , 'write_manifest' , fail )
389343 with pytest .warns (UserWarning , match = 'read-only scratch' ):
390- pquery , table , exported = _export (
344+ _ , table , exported = _export (
391345 graph , 'PREDICT ents.LABEL FOR EACH ents.ID' , 'no_manifest' )
392346 assert table .manifest is None
393347 assert not (out / 'no_manifest' / '_kumo_manifest.json' ).exists ()
394- assert (out / 'no_manifest' / '_SUCCESS' ).exists ()
395348 assert exported .num_rows == sum (_seeds (table )[1 ].values ())
396349
397350
398- def test_warehouse_sink_receives_no_sidecars (graph ):
399- graph_ , snapshot , _ , _ = graph
351+ def test_no_sidecar_for_warehouse_or_csv_exports (graph ):
352+ graph_ , snapshot , connector , out = graph
400353 table = kumo .PredictiveQuery (
401354 graph = graph_ , query = 'PREDICT ents.LABEL FOR EACH ents.ID'
402355 ).generate_training_table (snapshot )
403356 calls = []
404357
405358 def write_files (connector , table_name , files , write_mode , ** kwargs ):
406359 calls .append (kwargs )
407- return table_name
408360
409- connector = SimpleNamespace (
361+ warehouse = SimpleNamespace (
410362 source_type = 'SNOWFLAKE' ,
411363 sink_backend = SimpleNamespace (write_files = write_files ))
412- table .export (TrainingTableOutputConfig (connector , 'DB.SCHEMA.TT' ))
364+ table .export (TrainingTableOutputConfig (warehouse , 'DB.SCHEMA.TT' ))
413365 assert calls == [{'export_id' : f'{ table .tt_id } :tt-export-v1' }]
414-
415-
416- def test_csv_directory_export_gets_no_sidecar (graph ):
417- graph_ , snapshot , connector , out = graph
418- table = kumo .PredictiveQuery (
419- graph = graph_ , query = 'PREDICT ents.LABEL FOR EACH ents.ID'
420- ).generate_training_table (snapshot )
421366 table .export (TrainingTableOutputConfig (connector , 'rows.csv/' ))
422- names = {p .name for p in (out / 'rows.csv' ).iterdir ()}
423- assert '_SUCCESS' in names and '_kumo_manifest.json' not in names
367+ assert not (out / 'rows.csv' / '_kumo_manifest.json' ).exists ()
0 commit comments