pipeline/cli.py at main · proxectonos/pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import os
import argparse
from inspect import getmembers, isfunction
from methods import subprocesses

class ListMethodsAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string):
        print(getmembers(subprocesses, isfunction))
        print(help(subprocesses))
        parser.exit()

def build_parser(handlers):
    """
    Builds the argument parser and maps handlers to subcommands.
    :param handlers: A dictionary mapping action names to function objects.
    """
    parser = argparse.ArgumentParser(description="Nós NLP pipeline.")
    base_parser = argparse.ArgumentParser(add_help=False)
    base_parser.add_argument("--mode", "-m", default="jsonl", help=".txt or .jsonl output format")

    parser.add_argument("-lm", "--list_methods", nargs=0, action=ListMethodsAction)
    parser.add_argument("--eol", help="convert EOL to linux format", action=argparse.BooleanOptionalAction)

    subparsers = parser.add_subparsers(dest="action", help="choose one")

    # 1. MT Quelingua
    p = subparsers.add_parser("mt_quelingua", parents=[base_parser])
    p.add_argument("-s", "--source", type=str, required=True)
    p.add_argument("-t", "--target", type=str, required=True)
    p.add_argument("-cl", "--correct_lang_source", type=str, required=True)
    p.add_argument("-ot", "--output_tag", type=str, required=False, default="_quelingua")
    p.set_defaults(func=handlers['mt_quelingua'])

    # 2. Formatter
    p = subparsers.add_parser("formatter", parents=[base_parser])
    p.add_argument("-o", "--output", type=str, required=True)
    p.add_argument("--path", "-p", required=True)
    p.add_argument("-t", "--technique", default="regex")
    p.add_argument("-d", "--delimiter", default="\n\n\n")
    p.set_defaults(func=handlers['formatter'])

    # 3. Pipeline Tasks
    pipeline_cmds = ["encoder", "tokenizer", "detokenizer", "filter_lang", "pyplexity", "mt_transliteration"]
    for cmd in pipeline_cmds:
        p = subparsers.add_parser(cmd, parents=[base_parser])
        p.add_argument("--path", "-p", required=True)
        p.add_argument("-o", "--output", required=True)

        if cmd == "encoder":
            p.add_argument("-cat", "--categories", default=[])
            p.add_argument("-char", "--characters", default="")
            p.add_argument("-rmchar", "--remove-characters", default="")
            p.add_argument("-emo", "--emojies", action=argparse.BooleanOptionalAction, default=False)
        elif cmd == "filter_lang":
            p.add_argument("-f", "--filter_results_by_lang", default=False)
        elif cmd == "pyplexity":
            default_model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models/bigrams_modelo-gl-bigramas-merged.st")
            p.add_argument("-pm", "--path_model", default=default_model_path)
            p.add_argument("-s", "--score", action=argparse.BooleanOptionalAction, default=True)
            p.add_argument("-pl", "--perpl_limit", type=int, default=2000)
            p.add_argument("-r", "--remove_low_scores", action=argparse.BooleanOptionalAction, default=True)
            p.add_argument("pyplexity_args", nargs=argparse.REMAINDER)
        elif cmd == "mt_transliteration":
            p.add_argument("-q", "--quelingua", action=argparse.BooleanOptionalAction, default=False)
            p.add_argument("-f", "--field", default=None, help="Field name for jsonl files")
            #p.set_defaults(func=handlers['mt_transliteration'])

        p.set_defaults(func=handlers['pipeline'])

    # 4. Direct Tools
    p = subparsers.add_parser("normalize", parents=[base_parser])
    p.add_argument("-p", "--path", required=True)
    p.add_argument("--detokenize", action=argparse.BooleanOptionalAction, default=False)
    p.add_argument("--jsonl_field", action=argparse.BooleanOptionalAction, default=None)
    p.add_argument("-o", "--output", required=True)
    p.add_argument("-b", "--bel", action=argparse.BooleanOptionalAction ,default=False)
    p.add_argument("-e", "--exact", action=argparse.BooleanOptionalAction ,default=False)
    p.set_defaults(func=handlers['normalize'])

    p = subparsers.add_parser("recoglang", parents=[base_parser])
    p.add_argument("-p", "--path", required=True)
    p.set_defaults(func=handlers['recoglang'])

    p = subparsers.add_parser("fix_new_lines")
    p.add_argument("-p", "--path", required=True)
    p.set_defaults(func=handlers['fix_newlines'])

    p = subparsers.add_parser("jaccard", parents=[base_parser])
    p.add_argument("-p", "--path", required=True)
    p.add_argument("-o", "--output_file", action='store_true')
    p.add_argument("-md", "--max_documents", type=int)
    p.add_argument("-lt", "--length_threshold", type=int, default=3)
    p.add_argument("-lsh", "--lsh_threshold", type=float, default=0.8)
    p.add_argument("-gds", "--generate_deduplication_samples", action="store_true")
    p.set_defaults(func=handlers['jaccard'])

    p = subparsers.add_parser("deduplication", parents=[base_parser])
    p.add_argument("-p", "--path", required=True)
    p.add_argument("-o", "--output")
    p.add_argument("-ilf", "--input_lf", type=int, default=3)
    p.add_argument("-olf", "--output_lf", type=int, default=2)
    p.add_argument("-t", "--threshold", type=int, default=15)
    p.add_argument("-s", "--save_duplicates", action=argparse.BooleanOptionalAction, default=False)
    p.set_defaults(func=handlers['deduplication'])

    p = subparsers.add_parser("mt_deduplication", parents=[base_parser])
    p.add_argument("-s", "--source", required=True)
    p.add_argument("-t", "--target", required=True)
    p.add_argument("-f", "--field", default=None, help="Field name for jsonl files")
    p.add_argument("-d", "--save_duplicates", action=argparse.BooleanOptionalAction, default=False)
    p.set_defaults(func=handlers['mt_deduplication'])

    '''    p = subparsers.add_parser("mt_transliteration", parents=[base_parser])
        p.add_argument("-p", "--path", required=True)
        p.add_argument("-o", "--output", required=True)
        p.add_argument("-q", "--quelingua", action=argparse.BooleanOptionalAction, default=False)
        p.add_argument("-f", "--field", default=None, help="Field name for jsonl files")
        p.set_defaults(func=handlers['mt_transliteration'])
    '''

    return parser