reformat python files for uniform styling

Author: balysMorkunas

Diff for: CSharpExtractor/extract.py

@@ -12,23 +12,38 @@
 from subprocess import Popen, PIPE, STDOUT, call
 
 
-
 def get_immediate_subdirectories(a_dir):
-    return [(os.path.join(a_dir, name)) for name in os.listdir(a_dir)
-            if os.path.isdir(os.path.join(a_dir, name))]
+    return [
+        (os.path.join(a_dir, name))
+        for name in os.listdir(a_dir)
+        if os.path.isdir(os.path.join(a_dir, name))
+    ]
 
 
 TMP_DIR = ""
 
+
 def ParallelExtractDir(args, dir):
     ExtractFeaturesForDir(args, dir, "")
 
 
 def ExtractFeaturesForDir(args, dir, prefix):
-    command = ['dotnet', 'run', '--project', args.csproj,
-               '--max_length', str(args.max_path_length), '--max_width', str(args.max_path_width),
-               '--path', dir, '--threads', str(args.num_threads), '--ofile_name', str(args.ofile_name)]
-
+    command = [
+        "dotnet",
+        "run",
+        "--project",
+        args.csproj,
+        "--max_length",
+        str(args.max_path_length),
+        "--max_width",
+        str(args.max_path_width),
+        "--path",
+        dir,
+        "--threads",
+        str(args.num_threads),
+        "--ofile_name",
+        str(args.ofile_name),
+    ]
 
     # print command
     # os.system(command)
@@ -46,15 +61,16 @@ def ExtractFeaturesForDir(args, dir, prefix):
         if len(stderr) > 0:
             print(sys.stderr, stderr)
     else:
-        print(sys.stderr, 'dir: ' + str(dir) + ' was not completed in time')
+        print(sys.stderr, "dir: " + str(dir) + " was not completed in time")
         failed = True
         subdirs = get_immediate_subdirectories(dir)
         for subdir in subdirs:
-            ExtractFeaturesForDir(args, subdir, prefix + dir.split('/')[-1] + '_')
+            ExtractFeaturesForDir(args, subdir, prefix + dir.split("/")[-1] + "_")
     if failed:
         if os.path.exists(str(args.ofile_name)):
             os.remove(str(args.ofile_name))
 
+
 def ExtractFeaturesForDirsList(args, dirs):
     global TMP_DIR
     TMP_DIR = "./tmp/feature_extractor%d/" % (os.getpid())
@@ -64,7 +80,7 @@ def ExtractFeaturesForDirsList(args, dirs):
     try:
         p = multiprocessing.Pool(4)
         p.starmap(ParallelExtractDir, zip(itertools.repeat(args), dirs))
-        #for dir in dirs:
+        # for dir in dirs:
         #    ExtractFeaturesForDir(args, dir, '')
         output_files = os.listdir(TMP_DIR)
         for f in output_files:
@@ -73,12 +89,26 @@ def ExtractFeaturesForDirsList(args, dirs):
         shutil.rmtree(TMP_DIR, ignore_errors=True)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
 
     parser = ArgumentParser()
-    parser.add_argument("-maxlen", "--max_path_length", dest="max_path_length", required=False, default=8)
-    parser.add_argument("-maxwidth", "--max_path_width", dest="max_path_width", required=False, default=2)
-    parser.add_argument("-threads", "--num_threads", dest="num_threads", required=False, default=64)
+    parser.add_argument(
+        "-maxlen",
+        "--max_path_length",
+        dest="max_path_length",
+        required=False,
+        default=8,
+    )
+    parser.add_argument(
+        "-maxwidth",
+        "--max_path_width",
+        dest="max_path_width",
+        required=False,
+        default=2,
+    )
+    parser.add_argument(
+        "-threads", "--num_threads", dest="num_threads", required=False, default=64
+    )
     parser.add_argument("--csproj", dest="csproj", required=True)
     parser.add_argument("-dir", "--dir", dest="dir", required=False)
     parser.add_argument("-ofile_name", "--ofile_name", dest="ofile_name", required=True)
@@ -88,5 +118,5 @@ def ExtractFeaturesForDirsList(args, dirs):
         subdirs = get_immediate_subdirectories(args.dir)
         to_extract = subdirs
         if len(subdirs) == 0:
-            to_extract = [args.dir.rstrip('/')]
+            to_extract = [args.dir.rstrip("/")]
         ExtractFeaturesForDirsList(args, to_extract)

Diff for: Python150kExtractor/extract.py

@@ -10,23 +10,23 @@
 from pathlib import Path
 from sklearn import model_selection as sklearn_model_selection
 
-METHOD_NAME, NUM = 'METHODNAME', 'NUM'
+METHOD_NAME, NUM = "METHODNAME", "NUM"
 
 parser = argparse.ArgumentParser()
-parser.add_argument('--data_dir', required=True, type=str)
-parser.add_argument('--valid_p', type=float, default=0.2)
-parser.add_argument('--max_path_length', type=int, default=8)
-parser.add_argument('--max_path_width', type=int, default=2)
-parser.add_argument('--use_method_name', type=bool, default=True)
-parser.add_argument('--use_nums', type=bool, default=True)
-parser.add_argument('--output_dir', required=True, type=str)
-parser.add_argument('--n_jobs', type=int, default=multiprocessing.cpu_count())
-parser.add_argument('--seed', type=int, default=239)
+parser.add_argument("--data_dir", required=True, type=str)
+parser.add_argument("--valid_p", type=float, default=0.2)
+parser.add_argument("--max_path_length", type=int, default=8)
+parser.add_argument("--max_path_width", type=int, default=2)
+parser.add_argument("--use_method_name", type=bool, default=True)
+parser.add_argument("--use_nums", type=bool, default=True)
+parser.add_argument("--output_dir", required=True, type=str)
+parser.add_argument("--n_jobs", type=int, default=multiprocessing.cpu_count())
+parser.add_argument("--seed", type=int, default=239)
 
 
 def __collect_asts(json_file):
     asts = []
-    with open(json_file, 'r', encoding='utf-8') as f:
+    with open(json_file, "r", encoding="utf-8") as f:
         for line in f:
             ast = json.loads(line.strip())
             asts.append(ast)
@@ -42,22 +42,22 @@ def dfs(v):
 
         v_node = ast[v]
 
-        if 'value' in v_node:
+        if "value" in v_node:
             if v == node_index:  # Top-level func def node.
                 if args.use_method_name:
                     paths.append((stack.copy(), METHOD_NAME))
             else:
-                v_type = v_node['type']
+                v_type = v_node["type"]
 
-                if v_type.startswith('Name'):
-                    paths.append((stack.copy(), v_node['value']))
-                elif args.use_nums and v_type == 'Num':
+                if v_type.startswith("Name"):
+                    paths.append((stack.copy(), v_node["value"]))
+                elif args.use_nums and v_type == "Num":
                     paths.append((stack.copy(), NUM))
                 else:
                     pass
 
-        if 'children' in v_node:
-            for child in v_node['children']:
+        if "children" in v_node:
+            for child in v_node["children"]:
                 dfs(child)
 
         stack.pop()
@@ -84,12 +84,13 @@ def __raw_tree_paths(ast, node_index, args):
 
     tree_paths = []
     for (v_path, v_value), (u_path, u_value) in itertools.combinations(
-            iterable=tnodes,
-            r=2,
+        iterable=tnodes,
+        r=2,
     ):
         prefix, lca, suffix = __merge_terminals2_paths(v_path, u_path)
-        if (len(prefix) + 1 + len(suffix) <= args.max_path_length) \
-                and (abs(len(prefix) - len(suffix)) <= args.max_path_width):
+        if (len(prefix) + 1 + len(suffix) <= args.max_path_length) and (
+            abs(len(prefix) - len(suffix)) <= args.max_path_width
+        ):
             path = prefix + [lca] + suffix
             tree_path = v_value, path, u_value
             tree_paths.append(tree_path)
@@ -103,49 +104,49 @@ def __delim_name(name):
 
     def camel_case_split(identifier):
         matches = re.finditer(
-            '.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)',
+            ".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)",
             identifier,
         )
         return [m.group(0) for m in matches]
 
     blocks = []
-    for underscore_block in name.split('_'):
+    for underscore_block in name.split("_"):
         blocks.extend(camel_case_split(underscore_block))
 
-    return '|'.join(block.lower() for block in blocks)
+    return "|".join(block.lower() for block in blocks)
 
 
 def __collect_sample(ast, fd_index, args):
     root = ast[fd_index]
-    if root['type'] != 'FunctionDef':
-        raise ValueError('Wrong node type.')
+    if root["type"] != "FunctionDef":
+        raise ValueError("Wrong node type.")
 
-    target = root['value']
+    target = root["value"]
 
     tree_paths = __raw_tree_paths(ast, fd_index, args)
     contexts = []
     for tree_path in tree_paths:
         start, connector, finish = tree_path
 
         start, finish = __delim_name(start), __delim_name(finish)
-        connector = '|'.join(ast[v]['type'] for v in connector)
+        connector = "|".join(ast[v]["type"] for v in connector)
 
-        context = f'{start},{connector},{finish}'
+        context = f"{start},{connector},{finish}"
         contexts.append(context)
 
     if len(contexts) == 0:
         return None
 
     target = __delim_name(target)
-    context = ' '.join(contexts)
+    context = " ".join(contexts)
 
-    return f'{target} {context}'
+    return f"{target} {context}"
 
 
 def __collect_samples(ast, args):
     samples = []
     for node_index, node in enumerate(ast):
-        if node['type'] == 'FunctionDef':
+        if node["type"] == "FunctionDef":
             sample = __collect_sample(ast, node_index, args)
             if sample is not None:
                 samples.append(sample)
@@ -160,18 +161,18 @@ def __collect_all_and_save(asts, args, output_file):
     samples = parallel(func(ast, args) for ast in tqdm.tqdm(asts))
     samples = list(itertools.chain.from_iterable(samples))
 
-    with open(output_file, 'w') as f:
+    with open(output_file, "w") as f:
         for line_index, line in enumerate(samples):
-            f.write(line + ('' if line_index == len(samples) - 1 else '\n'))
+            f.write(line + ("" if line_index == len(samples) - 1 else "\n"))
 
 
 def main():
     args = parser.parse_args()
     np.random.seed(args.seed)
 
     data_dir = Path(args.data_dir)
-    trains = __collect_asts(data_dir / 'python100k_train.json')
-    evals = __collect_asts(data_dir / 'python50k_eval.json')
+    trains = __collect_asts(data_dir / "python100k_train.json")
+    evals = __collect_asts(data_dir / "python50k_eval.json")
 
     train, valid = sklearn_model_selection.train_test_split(
         trains,
@@ -182,12 +183,12 @@ def main():
     output_dir = Path(args.output_dir)
     output_dir.mkdir(exist_ok=True)
     for split_name, split in zip(
-            ('train', 'valid', 'test'),
-            (train, valid, test),
+        ("train", "valid", "test"),
+        (train, valid, test),
     ):
-        output_file = output_dir / f'{split_name}_output_file.txt'
+        output_file = output_dir / f"{split_name}_output_file.txt"
         __collect_all_and_save(split, args, output_file)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

Diff for: args.py

@@ -5,21 +5,51 @@ def read_args():
     parser = ArgumentParser()
 
     group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("-d", "--data", dest="data_path",
-                       help="path to preprocessed dataset")
-    group.add_argument("-l", "--load_path", dest="load_path",
-                       help="path to load model files", metavar="FILE")
+    group.add_argument(
+        "-d", "--data", dest="data_path", help="path to preprocessed dataset"
+    )
+    group.add_argument(
+        "-l",
+        "--load_path",
+        dest="load_path",
+        help="path to load model files",
+        metavar="FILE",
+    )
 
-    parser.add_argument("-m", "--model_path", dest="model_path",
-                        help="path to save and load checkpoints", metavar="FILE", required=False)
-    parser.add_argument("-s", "--save_path", dest="save_path",
-                        help="path to save model files", metavar="FILE", required=False)
+    parser.add_argument(
+        "-m",
+        "--model_path",
+        dest="model_path",
+        help="path to save and load checkpoints",
+        metavar="FILE",
+        required=False,
+    )
+    parser.add_argument(
+        "-s",
+        "--save_path",
+        dest="save_path",
+        help="path to save model files",
+        metavar="FILE",
+        required=False,
+    )
 
-    parser.add_argument("-t", "--test", dest="test_path",
-                        help="path to test file", metavar="FILE", required=False)
+    parser.add_argument(
+        "-t",
+        "--test",
+        dest="test_path",
+        help="path to test file",
+        metavar="FILE",
+        required=False,
+    )
 
-    parser.add_argument('-p', '--predict', dest='predict', type=str, default='java',
-                        help='starts prediction mode, argument is "cpp" or "java" dependin on language model')
-    parser.add_argument('--debug', action='store_true')
-    parser.add_argument('--seed', type=int, default=239)
+    parser.add_argument(
+        "-p",
+        "--predict",
+        dest="predict",
+        type=str,
+        default="java",
+        help='starts prediction mode, argument is "cpp" or "java" dependin on language model',
+    )
+    parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--seed", type=int, default=239)
     return parser.parse_args()

Diff for: baseline_tokenization/javalang/__init__.py

@@ -1,4 +1,3 @@
-
 from . import parser
 from . import parse
 from . import tokenizer