__version__='0.1.0'
debug=False
spreadsheet_encoding='utf-8'
known_spreadsheet_extensions=['.csv','.tsv','.ods','.xlsx','.xls',]
version_array=['-v','--version']
help_array=['-h','--help','/?','?']
import csv
try:
import pyexcel
except:
#print('pyexcel is not available, to read and write formats besides .csv, install with \'python -m pip install pyexcel pyexcel-xls pyexcel-xlsx pyexcel-ods pyexcel-ods3 pyexcel-odsr openpyxl\'')
pyexcel=None
import sys
import os
import tempfile
import datetime
import argparse
import json
home_path=os.path.expanduser('~')
def make_path_friendly(path):
if path.startswith(home_path):
path=path.replace(home_path,'~',1)
return path
def get_parent_folder(path):
return os.path.abspath(path+'/'+os.pardir)
csv_dialects=csv.list_dialects()#['unix','excel','excel-tab']
script_name=os.path.basename(sys.argv[0])
script_path=get_parent_folder(__file__)
def get_temp_folder(filename=None):
if not filename: return tempfile.gettempdir()+'/'+os.path.splitext(script_name)[0]
elif filename == '': return None
return tempfile.gettempdir()+'/'+os.path.splitext(script_name)[0]+'/'+filename
invalid_symbols_for_filenames=['\\','/',':','?','"','<','>','|','~','!','{','}','(',')','-','♥','☆',',','×','♪']
def make_filename_write_friendly(filename,ascii_only=True):
filename=filename.strip()
if ascii_only: filename=filename.encode('ascii',errors='replace').decode().replace('?','')
for symbol in invalid_symbols_for_filenames:
if symbol in filename:
filename=filename.replace(symbol,'_')
return filename
def read_file(filename,encoding='utf-8',as_json=False):
if not filename: return None
if not os.path.isfile(filename):
return None
print('reading',make_path_friendly(filename))
if encoding == 'binary':
with open(filename,'rb') as file:
return bytearray(file.read())
else:
with open(filename,'r',encoding=encoding) as file:
if as_json: return json.loads(file.read())
else: return file.read()
def write_file(filename,data,encoding='utf-8',as_json=False,force=False):
if not filename: return None
if not data: return None
if len(data) == 0: return None
if os.path.isfile(filename) and (not force):
print(filename,'already exists')
return False
if encoding == 'binary':
if isinstance(data,str):
data=data.encode('utf-8')
with open(filename,'wb') as file:
file.write(data)
else:
with open(filename, 'w',encoding=encoding) as file:
if as_json: json.dump(data,file,ensure_ascii=False,indent=4)
else: file.write(data)
if os.path.isfile(filename):
print('wrote',make_path_friendly(filename))
return True
return False
#accepts a spreadsheet path and returns either a list or character_names.csv as a Python dictionary
def read_spreadsheet(spreadsheet_path=None,csv_dialect=None,character_dictionary=False):
if character_dictionary == False:
spreadsheet=[]
else:
spreadsheet={}
if spreadsheet_path == None:
return spreadsheet
print('reading',os.path.basename(spreadsheet_path))
#open spreadsheet.csv
#check if input is a .csv
extension=os.path.splitext(spreadsheet_path)[1].lower()
if (extension == '.csv') or (extension == '.tsv'):
#https://docs.python.org/3/library/csv.html
with open(spreadsheet_path,'rt',encoding=spreadsheet_encoding,newline='',errors='strict') as file:
if csv_dialect == None:
if extension == '.csv': csv_file=csv.reader(file)
elif extension == '.tsv': csv_file=csv.reader(file,delimiter='\t')
else:
if extension == '.csv': csv_file=csv.reader(file,dialect=csv_dialect)
elif extension == '.tsv': csv_file=csv.reader(file,dialect=csv_dialect,delimiter='\t')
#"A csvfile must be an iterable of strings, each in the reader's defined csv format. A csvfile is most commonly a file-like object or list."
for index,csv_list in enumerate(csv_file):
if debug == True:
print(str(csv_list))
if character_dictionary == False:
spreadsheet.append([i.strip() for i in csv_list])
else:
if index == 0:
continue
if len(csv_list) > 1:
if csv_list[1] != None:
if csv_list[1].strip() != '':
spreadsheet[csv_list[0].strip()]=csv_list[1].strip()
else:
if character_dictionary == False:
#this returns a [[],[],[]] object where each inner list is a row
spreadsheet=pyexcel.get_array(file_name=spreadsheet_path, start_row=0)
for index,row in enumerate(spreadsheet):
for i,cell in enumerate(spreadsheet[index]):
if cell != None:
spreadsheet[index][i]=cell.strip()
if debug == True:
print(spreadsheet)
else:
#this returns dictionaries, a lot of them with duplicate data
records=pyexcel.get_records(file_name=spreadsheet_path)
if debug == True:
print(records)
headers=[]
#this relies on the python 3.7+ (cpython 3.6+) functionality of dictionaries being ordered to preserve the order of the headers
for key in records[0].keys():
headers.append(key)
for row in records:
if row[headers[1]] != None:
if row[headers[1]].strip() != '':
spreadsheet[row[headers[0]].strip()]=row[headers[1]].strip()
return spreadsheet
def write_spreadsheet(path,data,encoding=spreadsheet_encoding,csv_dialect=None,errorhandling='namereplace'):
if os.path.exists(path):
return None
if os.path.isdir(path):
print('cannot write spreadsheet to folder path',path)
return None
assert isinstance(path,str)
assert isinstance(data,list)
extension=path[-4:].lower()
if (extension == '.csv') or (extension == '.tsv'):
with open(path,'wt',encoding=encoding,newline='',errors='namereplace') as file:
if csv_dialect == None:
if extension == '.csv': csv_file=csv.writer(file)
elif extension == '.tsv': csv_file=csv.writer(file,delimiter='\t')
else:
if extension == '.csv': csv_file=csv.writer(file,dialect=csv_dialect)
elif extension == '.tsv': csv_file=csv.writer(file,dialect=csv_dialect,delimiter='\t')
for row in data:
csv_file.writerow(row)
else:
pyexcel.save_as(array=data,dest_file_name=path)
if os.path.isfile(path): print('wrote',path)
def is_ascii(a_string):
is_ascii=False
try:
a_string.encode('ascii')
is_ascii=True
except:
pass
return is_ascii
blacklist=[
'？？？',
'「……」',
'ＭＳ 明朝',
'ｵﾜﾀ＼(＾o＾)／',
'［',
'］',
'ＭＳ ゴシック',
'IF_MW01セピア',
'IF_MWB01nセピア',
'test_白',
'test_緑',
'test_赤',
'test_青',
'test_黄',
'test_青デカ',
'「！」',
'！！',
'「……？」',
]
def extract(input_json_filename):
input_json=read_file(input_json_filename,as_json=True)
spreadsheet=[]
if 'strings' not in input_json:
print('unrecognized json format')
return []
strings=input_json['strings']
#metadata_0=[] #"entry_idx" or "index"
#metadata_1=[] #"string_idx" or "offset"
#metadata_2=[] #"event_id" or "max_bytes"
#headers=['untranslated_texts','metadata']
headers_dialogue=['untranslated_texts','entry_idx','string_idx','event_id']
headers_gsdb_strings=['untranslated_texts','index','offset','max_bytes']
if strings[0].get('entry_idx') is not None:
headers=headers_dialogue
dialogue_mode=True
else:
headers=headers_gsdb_strings
dialogue_mode=False
spreadsheet.append(headers)
for text_dictionary in strings:
"""
"entry_idx": 188,
"string_idx": 3,
"event_idx": 1058,
"original": "MU_04",
"translation": "MU_04"
or
"index": 0,
"offset": 0,
"max_bytes": 4,
"original": "升䉄",
"translation": "升䉄"
"""
#entry_index=text_dictionary.get('entry_idx') #dialgue.json
#: text_dictionary.get('index') #gsdb_strings.json
if dialogue_mode:
entry_index=text_dictionary.get('entry_idx')
string_index=text_dictionary.get('string_idx')
event_index=text_dictionary.get('event_idx')
text=text_dictionary.get('original').strip()
if debug:
assert entry_index != None
assert string_index != None
assert event_index != None
row=[text,entry_index,string_index,event_index]
if debug: assert row[1] != 'None'
else:
index=text_dictionary.get('index')
offset=text_dictionary.get('offset')
max_bytes=text_dictionary.get('max_bytes')
text=text_dictionary.get('original').strip()
row=[text,index,offset,max_bytes]
if (not is_ascii(text)) and (text not in blacklist):
spreadsheet.append(row)
return spreadsheet
#column is an index, so reference it as column-1
def get_translated_data_from_row(row,column=0,minimum_column_index=4):
if (column != 0) and (column < len(row)):
assert column >= minimum_column_index
data=row[column].strip()
if data != '': return data
for i in reversed(row[minimum_column_index:]):
data=i.strip()
if data != '': return data
return ''
def postprocess(line):
line=line.strip()
return line
def insert(input_json_filename,spreadsheet_filename,csv_dialect=None,column=0):
""" updated_json=insert(input_json_filename=input.inputfile,spreadsheet_filename=input.spreadsheet,csv_dialect=input.csv_dialect)
"""
input_json=read_file(input_json_filename,as_json=True)
if input_json is None:
print('unable to read',input_json_filename)
raise Exception()
if 'strings' not in input_json:
print('unrecognized json format')
return []
strings=input_json['strings']
spreadsheet=read_spreadsheet(spreadsheet_path=spreadsheet_filename,csv_dialect=csv_dialect)
if spreadsheet is None:
print('unable to read',spreadsheet_filename)
raise Exception()
#headers_dialogue=['untranslated_texts','entry_idx','string_idx','event_id']
#headers_gsdb_strings=['untranslated_texts','index','offset','max_bytes']
if strings[0].get('entry_idx') is not None:
#headers=headers_dialogue
dialogue_mode=True
else:
#headers=headers_gsdb_strings
dialogue_mode=False
for row_number,row_data in enumerate(spreadsheet):
if row_number == 0: continue
text_from_spreadsheet=row_data[0]
translated_data_from_spreadsheet=get_translated_data_from_row(row_data,column=column,minimum_column_index=4)
if translated_data_from_spreadsheet == '':
continue
if dialogue_mode:
entry_index_spreadsheet=int(row_data[1])
string_index_spreadsheet=int(row_data[2])
event_index_spreadsheet=int(row_data[3])
for text_dictionary in strings:
entry_index_json=int(text_dictionary.get('entry_idx'))
string_index_json=int(text_dictionary.get('string_idx'))
event_index_json=int(text_dictionary.get('event_idx'))
if entry_index_spreadsheet != entry_index_json:
continue
if string_index_spreadsheet != string_index_json:
continue
if event_index_spreadsheet != event_index_json:
continue
raw_text_from_json=text_dictionary.get('original').strip()
assert text_from_spreadsheet == raw_text_from_json
text_dictionary['translation']=text_dictionary['translation'].replace(raw_text_from_json,translated_data_from_spreadsheet)
break
else:
index_spreadsheet=int(row_data[1])
offset_spreadsheet=int(row_data[2])
max_bytes_spreadsheet=int(row_data[3])
for text_dictionary in strings:
index_json=int(text_dictionary.get('index'))
offset_json=int(text_dictionary.get('offset'))
max_bytes_json=int(text_dictionary.get('max_bytes'))
if index_spreadsheet != index_json:
continue
if offset_spreadsheet != offset_json:
continue
if max_bytes_spreadsheet != max_bytes_json:
continue
raw_text_from_json=text_dictionary.get('original').strip()
assert text_from_spreadsheet == raw_text_from_json
text_dictionary['translation']=text_dictionary['translation'].replace(raw_text_from_json,translated_data_from_spreadsheet)
break
return input_json
def main():
# https://docs.python.org/3/library/argparse.html
# https://www.gnu.org/prep/standards/standards.html#Command_002dLine-Interfaces
cli=argparse.ArgumentParser(description='Extract and insert text into kirikiriz .txt.scn files after they have been extracted from archive.xp3. Any folders in the paths must already exist. For usage, \'python tool.py -h\' version='+__version__)
cli.add_argument('mode',help='must be extract or insert')
cli.add_argument('inputfile',help='the source file or folder to extract strings from and insert them into')
cli.add_argument('-s','--spreadsheet', help='the file name to read and write the extracted strings to and from, the first row is reserved for column headers, must be .csv if pyexcel is not installed with \'python -m pip install pyexcel pyexcel-xls pyexcel-xlsx pyexcel-ods pyexcel-ods3 pyexcel-odsr openpyxl==3.0.10\', for batches, add the spreadsheet extension to change the output format '+str(known_spreadsheet_extensions))
cli.add_argument('-cn','--character_names', help='a .csv or spreadsheet mapping the raw character name to a translation, the first row is reserved for column headers')
cli.add_argument('-o','--output', help='the output file name or folder for the resulting file, only used for mode=insert')
cli.add_argument('-c','--column', help='the column number in the spreadsheet to use as replacements, only used for mode=insert, starts from 1, column A is the 1st column, so enter 1, column D is the 4th column, so enter 4, column C is 3, 0 is a special flag that uses the right most column, default is '+str(0), default=0, type=int)
cli.add_argument('-w','--wordwrap', help='word wrap setting, enter the number of characters per line, word wrap assumes a maximum of 3 lines, default is '+str(50)+' sane values are 30-80 characters, if accurate word wrapping is enabled using wordwrap_font, this is instead interpreted as the maximum pixel length, sane values for max pixel length are 400+', default=50, type=int)
cli.add_argument('-wf','--wordwrap_font', help='optional, enable accurate word wrapping by providing the full path to the .ttf font that will display the text, requires the PIL image library at >= 8.x, python -m pip install pillow')
cli.add_argument('-wfe','--wordwrap_font_encoding', help='optional, the encoding of the .ttf font, can be unic for Unicode, sjis for shift-jis, big5, see more options at https://hugovk-pillow.readthedocs.io/en/stable/reference/ImageFont.html#PIL.ImageFont.truetype')
cli.add_argument('-wp','--wordwrap_point_size', help='optional, for accurate word wrapping, this is the point size to use with the .ttf font, default is '+str(12), default=12, type=int)
cli.add_argument('-cd','--csv_dialect', help='specify the csv dialect when reading spreadsheet.csv files, normal settings are used otherwise, ignored for non .csv formats, valid options are unix, excel, excel-tab')
cli.add_argument('-v','--version', help='print version information', action='store_true')
cli.add_argument('-d','--debug', help='print debug information', action='store_true')
cli.add_argument('-t','--test', help='read input and process data but do not write any output', action='store_true')
# https://stackoverflow.com/questions/4042452/display-help-message-with-python-argparse-when-script-is-called-without-any-argu
if len(sys.argv) == 1:
cli.parse_args(['--help'])
for i in sys.argv:
if i.lower() in version_array:
print(os.path.basename(sys.argv[0]),__version__)
exit(0)
for i in sys.argv:
if i.lower() in help_array:
cli.parse_args(['--help'])
#input is a class with a lot of variables that can be accessed as input.mode input.spreadsheet and so forth
input=cli.parse_args()
global debug
debug=input.debug
if not os.path.isfile(input.inputfile):
print(input.inputfile,'does not exist')
exit(1)
modes=['extract','insert']
input.mode=input.mode.lower()
if input.mode not in modes:
print('unrecognized mode',input.mode,'must be in',modes)
exit(1)
if not input.spreadsheet:
input.spreadsheet=input.inputfile+'.csv'
else:
assert os.path.sifile(input.spreadsheet)
if not input.output:
input.output=input.inputfile+'.translated.json'
if input.mode == 'extract':
if os.path.isfile(input.spreadsheet):
print(input.spreadsheet,'already exists')
exit(1)
elif input.mode == 'insert':
if not os.path.isfile(input.spreadsheet):
print(input.spreadsheet,'does not exists')
exit(1)
if os.path.isfile(input.output):
print(input.output,'already exists')
exit(1)
if input.mode == 'extract':
extracted_lines_as_spreadsheet=extract(input_json_filename=input.inputfile)
if not extracted_lines_as_spreadsheet or (len(extracted_lines_as_spreadsheet) < 2): #len is 1 due to header
print('no texts found')
exit(0)
if debug:
print(extracted_lines_as_spreadsheet)
if not input.test:
write_spreadsheet(input.spreadsheet,extracted_lines_as_spreadsheet,csv_dialect=input.csv_dialect)
elif input.mode == 'insert':
updated_json=insert(input_json_filename=input.inputfile,spreadsheet_filename=input.spreadsheet,csv_dialect=input.csv_dialect,column=input.column)
if not updated_json:
print('error inserting texts')
exit(1)
if not input.test:
write_file(input.output,updated_json,as_json=True)
if __name__ == '__main__':
main()
