-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathcsvpaths.py
More file actions
1693 lines (1606 loc) · 61.3 KB
/
Copy pathcsvpaths.py
File metadata and controls
1693 lines (1606 loc) · 61.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""CsvPaths' intent is to help you manage and automate your use
of the CsvPath library. it makes it easier to scale your CSV quality control."""
import os
import traceback
import atexit
from uuid import uuid4
from abc import ABC, abstractmethod
from typing import Any, NewType, Optional
from datetime import datetime
from .managers.errors.error import Error
from .managers.errors.error_comms import ErrorCommunications
from .managers.errors.error_manager import ErrorManager
from .managers.errors.error_collector import ErrorCollector
from .util.config import Config
from .util.log_utility import LogUtility as lout
from .util.metadata_parser import MetadataParser
from .util.exceptions import InputException, CsvPathsException
from .util.references.reference_parser import ReferenceParser
from .util.run_home_maker import RunHomeMaker
from .managers.paths.paths_manager import PathsManager
from .managers.files.file_manager import FileManager
from .managers.results.results_manager import ResultsManager
from .managers.results.result import Result
from .util.box import Box
from . import CsvPath
# types for clarity
Reference = NewType("Reference", str)
class CsvPathsCoordinator(ABC):
"""
@private
This abstract class defines callbacks for CsvPath instances to
broadcast state to their siblings through CsvPaths. A CsvPath
instance might stop the entire run, rather than each CsvPath
instance needing to contain the same logic that stops their
participation in a run.
"""
@abstractmethod
def stop_all(self) -> None: # pragma: no cover
"""
@private
Stops every CsvPath instance in a run"""
@abstractmethod
def fail_all(self) -> None: # pragma: no cover
"""
@private
Fails every CsvPath instance in a run"""
@abstractmethod
def skip_all(self) -> None: # pragma: no cover
"""
@private
skips the line for every CsvPath instance in a run"""
@abstractmethod
def advance_all(self, lines: int) -> None: # pragma: no cover
"""
@private
advances every CsvPath instance in a run"""
class CsvPaths(CsvPathsCoordinator, ErrorCollector):
#
# METRICS supports OTLP. it will often not be used, but
# we'll check it at shutdown just in case.
#
METRICS = None
METRICS_WRAP_REG = False
WRAPPED_UP = False
@classmethod
def _wrap_up_metrics(cls) -> None:
if cls.METRICS:
try:
cls.METRICS.logger().debug(
f"csvpaths.wrapping up: shutting down metrics: {cls.METRICS}"
)
cls.METRICS.provider.force_flush()
cls.METRICS = None
except Exception:
print(traceback.print_exc())
"""
a CsvPaths instance manages applying any number of csvpaths
to any number of files. CsvPaths applies sets of csvpaths
to a given file, on demand. Think of CsvPaths as a session
object. It gives you a way to manage files, csvpaths, and
the results generated by applying paths to files. It is not
intended for concurrent use. If you need multiple threads,
create multiple CsvPaths instances.
"""
# pylint: disable=too-many-instance-attributes
def __init__(
self,
*,
delimiter=",",
quotechar='"',
skip_blank_lines=True,
print_default=True,
project_context=None,
project=None,
# config: Config = None,
):
if CsvPaths.METRICS_WRAP_REG is False:
atexit.register(CsvPaths._wrap_up_metrics)
CsvPaths.METRICS_WRAP_REG = True
#
#
#
self._config = Config()
#
# the project and project context disambiguate loggers in a static multi-thread/
# multi-user env. they are also used in metrics other places. if the context isn't
# we use a uuid to make sure we have a unique name until hopefully we get a more
# specific name. when we reset the project or project context we'll reup the logger
#
# the expectation is that FlightPath Server will use API key as project context.
#
self._project = project if project else "project"
self._project_context = project_context if project_context else "csvpaths"
#
# in a few cases, mainly s3 and sftp connection or config sharing
# we need to pass some state around. it's ugly, but logically not
# terrible and better than other options bar major refactoring.
#
box = Box()
box.add(Box.CSVPATHS_CONFIG, self._config)
#
# managers centralize activities, offer async potential, and
# are where integrations hook in. ErrorManager functionality
# must be available in CsvPath too. The others are CsvPaths
# only.
#
self._paths_manager = None
self._file_manager = None
self._results_manager = None
self._ecoms = None
self._error_manager = None
self._set_managers()
#
#
#
self.print_default = print_default
""" @private """
self.delimiter = delimiter
""" @private """
self.quotechar = quotechar
""" @private """
self.skip_blank_lines = skip_blank_lines
""" @private """
self.current_matcher: CsvPath = None
""" @private """
#
# all logs in a project (or stand-alone csvpaths) have the same
# log file path, as configured in config/config.ini. however, they
# have different names for the logger instances they use because
# it is possible, and happens in FlightPath Server, that the loggers
# are held by name in a static context that has multiple projects
# and so needs multiple file paths held by individual loggers.
#
self._config._assure_logs_path()
self._logger = None
#
#
#
# self.info_dump()
self._errors = []
# coordinator attributes
self._stop_all = False
self._fail_all = False
self._skip_all = False
self._advance_all = 0
#
# TODO: we probably don't need all three of these
#
self._current_run_time = None
self._run_time_str = None
self._last_run_dir = None
#
#
#
self.named_paths_name = None
""" @private """
self.named_file_name = None
""" @private """
#
# this metadata is generated at the run start. it is
# the coordinating metadata for the run in the sense
# that its UUID is the correlation ID all metadata
# objects from the same run use to understand their
# relationship.
#
self.run_metadata = None
#
# for many purposes csvpaths can clean up at the end of
# runs. in some cases you may want to do it manually
# later. e.g. if you are doing a run and then want to
# inspect or use the results in another run you might
# want to keep the connections live. when ready, call
# wrap_up(). defaults to True.
#
self._wrap_up_automatically = True
""" @private """
#
# adding a reference to the csvpaths in the run as they are created
# we'll clean this up in the wrap-up. not loving this, but it
# doesn't seem unreasonable and it is needed for functions (etc?)
# that need to know about the run during the run; e.g. parquet()
#
self._csvpath_instances = []
#
# any listeners added here will be added to the internal_listeners of
# the csvpath instances that run.
#
self.dynamic_csvpath_error_listeners = []
#
#
#
# this log line didn't add especial value. but it did set the logger
# to a certain log file before we had a chance to do a cd-and-reload
# to switch into a particular project. the fix isn't just to remove the
# line, it's also to make the logger travel with the config used. but
# removing the log line is easy enough and we don't lose much. And the
# real fix was already in set_config_path_and_reload() -- it was just
# releasing/renewing the logger.
#
# self.logger.info(
# f"Initialized CsvPaths: {self} in thread: {threading.current_thread()}"
# )
@property
def logger(self):
if self._logger is None:
self._logger = lout.logger(self)
return self._logger
@logger.setter
def logger(self, ler) -> None:
self._logger = ler
def set_config_path_and_reload(self, config_path: str) -> None:
self.config.set_config_path_and_reload(config_path)
#
# update anything we need to, e.g. logs. that may be all for now.
#
self.config._assure_logs_path()
self.logger = lout.logger(self)
def info_dump(self) -> None:
string = self.info_dump_string
self.logger.info(string)
@property
def info_dump_string(self) -> str:
lstrs = "Active integrations:"
intgs = self.config.get(section="listeners", name="groups")
if not isinstance(intgs, list):
intgs = intgs.split(",")
for i, _ in enumerate(intgs):
lstrs = f"{lstrs}\n - {_}"
subs = self.config.get(section="config", name="allow_var_sub")
subs = (
self.config.get(section="config", name="var_sub_source")
if str(subs).strip().lower() in ["yes", "true"]
else "none"
)
if subs not in ["env", "none"]:
subs = f"{subs} with keys:"
e = self.config.config_env.env
if e is not None:
for k, v in e.items():
subs = f"{subs}\n - {k}"
cache = self.config.get(section="cache", name="path")
cache = cache if self.config.get(section="cache", name="use_cache") else "none"
logstr = self.logger if self.logger else "pending"
logpath = ""
for h in self.logger.handlers:
try:
logpath = f"{logpath}\n {self.logger} - {h.baseFilename}"
except Exception:
...
return f"""
Context: {self.project_context}.{self.project}
Logger: {logstr}
Configured log path: {self.config.get(section="logging", name="log_file")}
Current log paths: {logpath}
Config file: {self.config.configpath}
Var sub source: {self.config.get(section="config", name="var_sub_source")}
Var subs: {subs}
Errors:
- csvpath: {self.config.get(section="errors", name="csvpath")}
- csvpaths: {self.config.get(section="errors", name="csvpaths")}
Named-files: {self.config.get(section="inputs", name="files")}
Named-paths: {self.config.get(section="inputs", name="files")}
Archive: {self.config.get(section="results", name="archive")}
Cache: {cache}
{lstrs}
"""
def _set_managers(self) -> None:
self.paths_manager = PathsManager(csvpaths=self)
self.file_manager = FileManager(csvpaths=self)
self.ecoms = ErrorCommunications(csvpaths=self)
self.error_manager = ErrorManager(csvpaths=self)
#
# we take a bit more care with resman because it may have listeners
# that are set programmatically. other managers deal with their
# registrars differently and don't have the problem of maintaining
# lists of listeners at the csvpaths level
#
resman = ResultsManager(csvpaths=self)
if self.results_manager:
resman.dynamic_result_listeners = (
self.results_manager.dynamic_result_listeners
)
resman.dynamic_results_listeners = (
self.results_manager.dynamic_results_listeners
)
resman.dynamic_run_listeners = self.results_manager.dynamic_run_listeners
self.results_manager = resman
@property
def project(self) -> str:
return self._project
@project.setter
def project(self, name: str) -> None:
#
# when we change projects we need to change names. we don't
# expect to do this much because we don't expect csvpaths to be
# long-lived. nevertheless.
#
cname = lout.logger_name(self)
self._project = name
new_cname = lout.logger_name(self)
if cname != new_cname:
self.logger = lout.logger(self)
@property
def project_context(self) -> str:
return self._project_context
@project_context.setter
def project_context(self, name: str) -> None:
#
# when we change projects we need to change names. we don't
# expect to do this much because we don't expect csvpaths to be
# long-lived. nevertheless.
#
cname = lout.logger_name(self)
self._project_context = name
new_cname = lout.logger_name(self)
if cname != new_cname:
self.logger = lout.logger(self)
@property
def wrap_up_automatically(self) -> bool:
return self._wrap_up_automatically
@wrap_up_automatically.setter
def wrap_up_automatically(self, auto: bool) -> None:
self._wrap_up_automatically = auto
@property
def ecoms(self) -> ErrorCommunications:
"""@private"""
return self._ecoms
@ecoms.setter
def ecoms(self, ec: ErrorCommunications) -> None:
"""@private"""
self._ecoms = ec
@property
def last_run_dir(self) -> str:
return self._last_run_dir
@property
def file_manager(self) -> FileManager:
return self._file_manager
@file_manager.setter
def file_manager(self, m: FileManager) -> None:
self._file_manager = m
@property
def results_manager(self) -> ResultsManager:
return self._results_manager
@results_manager.setter
def results_manager(self, m: ResultsManager) -> None:
self._results_manager = m
@property
def paths_manager(self) -> PathsManager:
return self._paths_manager
@paths_manager.setter
def paths_manager(self, m: PathsManager) -> None:
self._paths_manager = m
@property
def error_manager(self) -> ErrorManager:
return self._error_manager
@error_manager.setter
def error_manager(self, em: ErrorManager) -> None:
if em.csvpaths is None:
raise Exception("CsvPaths cannot be None")
self._error_manager = em
@property
def current_run_time(self) -> datetime:
maker = RunHomeMaker(self)
return maker.current_run_time
def csvpath(self) -> CsvPath:
"""Gets a CsvPath object primed with a reference to this CsvPaths"""
path = CsvPath(
csvpaths=self,
delimiter=self.delimiter,
quotechar=self.quotechar,
skip_blank_lines=self.skip_blank_lines,
#
# in the usual case we don't want csvpaths and its csvpath children
# to share the same config. sharing doesn't offer much. the flexibility
# of having separate configs is valuable.
#
# config=None,
print_default=self.print_default,
#
# we don't use this error manager reference atm.
#
error_manager=self.error_manager,
project=self.project,
project_context=self.project_context,
)
if path.config.configpath != self.config.configpath:
path.config.set_config_path_and_reload(self.config.configpath)
path.logger = None
for _ in self.dynamic_csvpath_error_listeners:
path.error_manager.add_internal_listener(_)
path.run_dir = self._last_run_dir
return path
def stop_all(self) -> None: # pragma: no cover
self._stop_all = True
def fail_all(self) -> None: # pragma: no cover
self._fail_all = True
def skip_all(self) -> None: # pragma: no cover
self._skip_all = True
def advance_all(self, lines: int) -> None: # pragma: no cover
self._advance_all = lines
@property
def errors(self) -> list[Error]: # pylint: disable=C0116
"""@private
generally you should be looking at results_manager or error_manager for errors.
"""
return self._errors
def collect_error(self, error: Error) -> None: # pylint: disable=C0116
"""@private"""
if not self.has_error(error):
self._errors.append(error)
def has_errors(self) -> bool: # pylint: disable=C0116
"""@private
generally you should be looking at results_manager or error_manager for errors.
"""
return len(self._errors) > 0
def has_error(self, e: Error) -> bool:
for _ in self.errors:
if _.equals(e):
return True
return False
@property
def config(self) -> Config: # pylint: disable=C0116
"""@private"""
if not self._config:
self._config = Config() # pragma: no cover
return self._config
#
# this is the preferred way to update config. it is preferred because
# csvpath and csvpaths work off the same config file, even though they,
# in some cases, have separate keys. if you update the config directly
# before a run starts using the CsvPaths's Config you have to remember
# to save and reload for it to effect both CsvPaths and CsvPath. this
# method does the save and reload every time.
#
def add_to_config(self, section, key, value) -> None:
"""@private"""
self.config.add_to_config(section=section, key=key, value=value)
self.config.save_config()
self.config.reload()
self._set_managers()
# =========================
# cleanup, wrapup calls
# =========================
def clear_run_coordination(self) -> None:
"""@private
run coordination is the set of signals that csvpaths send to affect
one another through the CsvPaths instance"""
self._stop_all = False
self._fail_all = False
self._skip_all = False
self._advance_all = 0
self._current_run_time = None
self._run_time_str = None
self.logger.debug("Cleared run coordination")
#
# we do not currently use the paths param. what was the purpose?
#
def clean(self, *, paths=None) -> None:
"""@private
at this time we do not recommend reusing CsvPaths, but it is doable
you should clean before reuse unless you want to accumulate results."""
#
# adding this Config() refresh breaks several tests, indicating -- i think --
# mainly that 1) we have some iffy tests that don't act like real use cases
# and 2) we sorta say that CsvPaths shouldn't be reused, but sorta act like
# we solved that so it can be, but aren't fully safe yet or at least haven't
# proved it yet.
#
# regardless, for now, don't reup the Config.
#
# self._config = Config()
#
self._set_managers()
self.clear_run_coordination()
self._errors = []
self.named_file_name = None
self.current_matcher = None
self.named_paths_name = None
self.run_metadata = None
self._logger = None
#
# clear csvpath references. these are collected as the run progresses.
#
self._csvpath_instances = []
def wrap_up(self) -> None:
#
# leave clean() and clear_run_coordination() in case there
# is some reason we might need to see the residual. those are
# called at top of run.
#
# cleanup shared connections, etc.
#
# CsvPaths.WRAPPED_UP is needed because if we call wrap_up()
# twice on the same thread another CsvPaths may have been
# started and cleaning up the contents of the thread's box
# could remove their stuff, no longer ours. We see this in
# unit tests, but in principle we could see it anywhere.
#
if CsvPaths.WRAPPED_UP is True:
return
CsvPaths.WRAPPED_UP = True
box = Box()
ds = []
for k, v in box.get_my_stuff().items():
if hasattr(v, "close"):
ds.append(k)
for _ in ds:
try:
v = box.get(_)
v.close()
box.remove(_)
except Exception as e:
msg = f"Error in wrapping up: {e}"
self.error_manager.handle_error(source=self, msg=msg)
if self.ecoms.do_i_raise():
raise CsvPathsException(msg)
self._logger = None
def __del__(self) -> None:
try:
self.wrap_up()
except Exception:
print(traceback.format_exc())
finally:
lout.release_logger(self)
def _trim_archive_if(self, home: str) -> str:
archive = self.config.get(section="results", name="archive")
if home.startswith(archive):
home = home[len(archive) + 1 :]
return home
def _trim_results_name_if(self, name: str, home: str) -> str:
if home.startswith(name):
home = home[len(name) + 1 :]
return home
def _make_run_reference(self, pathsname: str, crt: str) -> str:
return f"${pathsname}.results.{self._trim_results_name_if(pathsname, self._trim_archive_if(crt))}"
# =========================
# prep csvpath children
# =========================
def csvpath_instances(self) -> list[CsvPath]:
return self._csvpath_instances
def _load_csvpath(
self,
*,
csvpath: CsvPath,
path: str,
file: str,
pathsname: str = None,
filename,
by_line: bool = False,
crt: str,
index: int = -1,
) -> None:
#
# collecting for downstream access. prefer to not use this, but needed for some
# corner cases.
#
self._csvpath_instances.append(csvpath)
#
# file is the physical file (+/- if preceding mode) filename is the named-file name
#
self.logger.debug("Beginning to load csvpath %s with file %s", path, file)
csvpath.named_paths_name = pathsname
self.named_paths_name = pathsname
csvpath.named_file_name = filename
self.named_file_name = filename
#
# by_line==True means we are starting a run that is breadth-first ultimately using
# next_by_line(). by_line runs cannot be source-mode preceding and have different
# semantics around csvpaths influencing one another.
#
# we strip comments from above the path so we need to extract them first
#
path = MetadataParser(self).extract_metadata(instance=csvpath, csvpath=path)
identity = csvpath.identity
self.logger.debug("Csvpath %s after metadata extract: %s", identity, path)
#
# update the run settings using the metadata fields we just collected
#
csvpath.update_settings_from_metadata()
#
# historic note: file references are resolved in the filemanager. and we don't care
# for if we are using source-mode preceding for that.
#
if csvpath.data_from_preceding is True and index > 0:
#
# if index == 0 we can't look to the preceding csvpath in the run.
# if it is -1 we didn't pass in an index -- possibly because by_line.
#
if by_line is True:
raise CsvPathsException(
"Breadth-first runs do not support source-mode preceding because each line of data flows through each csvpath in order already"
)
#
# we are in source-mode: preceding that means we ignore the original data file path and
# instead use the data.csv from the preceding csvpath. that is, assuming there is a
# preceding csvpath and it created and saved data.
#
# find the preceding csvpath in the named-paths
#
# we may have a reference like: $sourcemode.csvpaths.source2:from. if so
# we just need the named-paths name.
#
# cannot do this anymore because ref templates.
# resman should get the whole reference
#
if pathsname.startswith("$"):
ref = ReferenceParser(pathsname, csvpaths=self)
idname = self.paths_manager._get_from_names(ref.root_major, identity)[0]
idnames = self.paths_manager.get_identified_path_names_in(pathsname)
i = idnames.index(idname)
identity = idnames[i - 1]
file = f"{crt}{os.sep}{identity}{os.sep}data.csv"
self.logger.debug("csvpaths.load_csvpath: idnames: %s", idnames)
self.logger.debug("csvpaths.load_csvpath: idname: %s", idname)
self.logger.debug("csvpaths.load_csvpath: identity: %s", identity)
self.logger.debug("csvpaths.load_csvpath: ref: %s", ref)
self.logger.debug("csvpaths.load_csvpath: pathsname: %s", pathsname)
self.logger.debug("csvpaths.load_csvpath: file: %s", file)
else:
#
# not a reference
#
result = self.results_manager.get_last_named_result(
name=pathsname, before=csvpath.identity
)
if result is not None:
# get its data.csv path for this present run
# swap in that path for the regular origin path
file = result.data_file_path
#
# this was added when the mode was created. it duplicates info available in manifests.
#
csvpath.metadata["source-mode-source"] = file
self.logger.info(
"Csvpath identified as %s uses last csvpath's data.csv at %s as source",
csvpath.identity,
file,
)
else:
self.logger.warning(
"No preceding data file for csvpath %s running in source-mode",
csvpath.identity,
)
f = path.find("[")
self.logger.debug("Csvpath matching part starts at char # %s", f)
apath = f"${file}{path[f:]}"
self.logger.info("Parsing csvpath %s", apath)
csvpath.parse(apath)
#
# ready to run. time to register the run. this is separate from
# the run_register.py (ResultsRegister) event
#
self.logger.debug("Done loading csvpath")
def _get_named_paths(self, pathsname: str) -> list:
if pathsname is None:
raise ValueError("Named-paths name cannot be None")
paths = self.paths_manager.get_named_paths(pathsname)
if paths is None:
raise InputException(f"No named-paths found for {pathsname}")
if not isinstance(paths, list):
raise InputException(
f"Named-paths group {pathsname} must be represented as a list[str]"
)
if len(paths) == 0:
raise InputException(f"Named-paths group {pathsname} is empty")
if "" in paths:
raise InputException(
f"Named-paths group {pathsname} has one or more empty csvpaths"
)
return paths
# =========================
# main functions
# =========================
#
# a filename pointer is typically a named-file name. however, it can be a reference.
# when it is a reference it will typically be to a specific csvpath's output, data.csv.
# it can also be to unmatched.csv. the reference can end in :data or :unmatched to
# specifically require those files. it can also just point to the csvpath, in which
# case data.csv is the default.
#
def collect_paths(
self,
*,
pathsname: str,
filename: str,
template: str = None,
extra_data: Optional[dict[str, str]] = None,
) -> Reference:
"""
Sequentially does a CsvPath.collect() on filename for every named-path in the
specified named-paths group. if a file reference is passed in, we iterate on
every concrete data file selected by the reference. unlike the named-paths
iteration, each file we iterate on becomes its own call to collect_paths. lines
are collected into a results object, not returned.
"""
files = self.file_manager.get_named_file(filename)
if files is None:
raise InputException(f"No named-file found for {filename}")
#
# if we came in with a specific file pointer we want to strip it down
# the more generic form now that we have the file.
#
if filename.endswith(":data"):
filename = filename[0:-5]
elif filename.endswith(":unmatched"):
filename = filename[0:-10]
if isinstance(files, list):
for file in files:
ref = self._collect_paths(
pathsname=pathsname,
filename=filename,
template=template,
file=file,
extra_data=extra_data,
)
else:
ref = self._collect_paths(
pathsname=pathsname,
filename=filename,
template=template,
file=files,
extra_data=extra_data,
)
#
# absolute reference to the results
#
return ref
def _collect_paths(
self,
*,
pathsname: str,
file: str,
filename: str,
template: str = None,
extra_data: Optional[dict[str, str]] = None,
) -> Reference:
#
# if template is None we need to go find any template that was given when
# the named-paths were loaded.
#
paths = self._get_named_paths(pathsname)
if template is None:
template = self.paths_manager.get_template_for_paths(pathsname)
self.logger.info(
"Prepping %s and %s with template %s", filename, pathsname, template
)
self.clean(paths=pathsname)
self.logger.info(
"Beginning collect_paths %s with %s paths using template %s",
pathsname,
len(paths),
template,
)
#
# run identification and directories created here
#
maker = RunHomeMaker(self)
crt = maker.get_run_dir(
paths_name=pathsname, file_name=filename, template=template
)
#
# capture the last run dir for the benefit of the caller
#
self._last_run_dir = crt
results = []
#
# adding uuid for the run as a whole
run_uuid = uuid4()
#
# run starts here
#
self.run_metadata = self.results_manager.start_run(
run_dir=crt,
pathsname=pathsname,
filename=filename,
file=file,
run_uuid=run_uuid,
method="collect_paths",
template=template,
extra_data=extra_data,
)
#
#
#
for i, path in enumerate(paths):
csvpath = self.csvpath()
if not csvpath.will_run:
continue
result = Result(
csvpath=csvpath,
file_name=filename,
paths_name=pathsname,
run_index=i,
run_time=self.current_run_time,
run_dir=crt,
run_uuid=run_uuid,
method="collect_paths",
template=template,
)
# casting a broad net because if "raise" not in the error policy we
# want to never fail during a run
try:
self._load_csvpath(
csvpath=csvpath,
path=path,
file=file,
pathsname=pathsname,
filename=filename,
crt=crt,
index=i,
)
#
# if run-mode: no-run we skip ahead without saving results
#
if not csvpath.will_run:
continue
#
# the add has to come after _load_csvpath because we need the identity or index
# to be stable and the identity is found in load, if it exists.
#
self.results_manager.add_named_result(result)
lines = result.lines
self.logger.debug("Collecting lines using a %s", type(lines))
csvpath.collect(lines=lines)
if lines is None:
self.logger.error( # pragma: no cover
"Unexpected None for lines after collect_paths: file: %s, match: %s",
file,
csvpath.match,
)
#
# TODO: unmatched needs additional support for streaming very large files
#
result.unmatched = csvpath.unmatched
except Exception as ex: # pylint: disable=W0718
if self.error_manager.csvpaths is None:
raise Exception("ErrorManager's CsvPaths cannot be None")
self.error_manager.handle_error(source=self, msg=f"{ex}")
if self.ecoms.do_i_raise():
self.results_manager.save(result)
raise
self.results_manager.save(result)
results.append(result)
#
# run ends here
#
self.results_manager.complete_run(
run_dir=crt, pathsname=pathsname, results=results
)
#
# update/write run manifests here
# - validity (are all paths valid)
# - paths-completeness (did they all run and complete)
# - method (collect, fast_forward, next)
# - timestamp
#
self.clear_run_coordination()
self.logger.info(
"Completed collect_paths %s with %s paths", pathsname, len(paths)
)
if self.wrap_up_automatically:
self.wrap_up()
#
# the run home is the most specific reference we can return
#
# return f"${pathsname}.results.{crt}"
ret = self._make_run_reference(pathsname=pathsname, crt=crt)
return ret
def fast_forward_paths(
self,
*,
pathsname: str,
filename: str,
template: str = None,
extra_data: Optional[dict[str, str]] = None,
) -> Reference:
"""
Sequentially does a CsvPath.fast_forward() on filename for every named path. No matches are collected.
"""
files = self.file_manager.get_named_file(filename)
if files is None:
raise InputException(f"No named-file found for {filename}")
ref = None
if isinstance(files, list):
for file in files:
ref = self._fast_forward_paths(
pathsname=pathsname,
filename=filename,
template=template,
file=file,
extra_data=extra_data,
)
else:
ref = self._fast_forward_paths(
pathsname=pathsname,
filename=filename,
template=template,
file=files,
extra_data=extra_data,
)
return ref
def _fast_forward_paths(
self,
*,
pathsname: str,
file: str,
filename: str,