Showing
1 changed file
with
2 additions
and
2 deletions
| ... | @@ -59,7 +59,7 @@ def jobs(repo, args): | ... | @@ -59,7 +59,7 @@ def jobs(repo, args): |
| 59 | for line, code in mod.diff_parsed["deleted"]: | 59 | for line, code in mod.diff_parsed["deleted"]: |
| 60 | deleted.extend(args.tokenizer.tokenize(code)) | 60 | deleted.extend(args.tokenizer.tokenize(code)) |
| 61 | 61 | ||
| 62 | - if len(added) + len(deleted) <= args.max_source_length: | 62 | + if added and deleted and len(added) + len(deleted) <= args.max_source_length - 3: |
| 63 | with jsonlines.open(args.output_file, mode="a") as writer: | 63 | with jsonlines.open(args.output_file, mode="a") as writer: |
| 64 | writer.write( | 64 | writer.write( |
| 65 | { | 65 | { |
| ... | @@ -105,7 +105,7 @@ def main(args): | ... | @@ -105,7 +105,7 @@ def main(args): |
| 105 | ) | 105 | ) |
| 106 | write_jsonl( | 106 | write_jsonl( |
| 107 | data[int(n_data * 0.9):int(n_data * 0.95)], | 107 | data[int(n_data * 0.9):int(n_data * 0.95)], |
| 108 | - path=args.output_dir, mode='validation' | 108 | + path=args.output_dir, mode='valid' |
| 109 | ) | 109 | ) |
| 110 | write_jsonl( | 110 | write_jsonl( |
| 111 | data[int(n_data * 0.95):], | 111 | data[int(n_data * 0.95):], | ... | ... |
-
Please register or login to post a comment