KNIME and UTF-8 characters with CSV between Python nodes with Pandas and Arrow

KNIME and UTF-8 characters with CSV between Python nodes with Pandas and Arrow

import knime.scripting.io as knio

import pyarrow as pa
import pyarrow.csv as csv

import pandas as pd

var_import_csv_file = knio.flow_variables['v_path_csv_file']

# import CSV file with Arrow
df_arrow = csv.read_csv(var_import_csv_file
, read_options = csv.ReadOptions(encoding='utf8')
, parse_options= csv.ParseOptions(delimiter='|', quote_char = '"')
)

var_csv_from_arrow_path = knio.flow_variables['context.workflow.data-path'] + "from_arrow_csv_file.csv"
knio.flow_variables['var_csv_from_arrow_path']= var_csv_from_arrow_path

csv.write_csv(df_arrow, var_csv_from_arrow_path)

# import CSV file with Pandas ------------------------------------------------
df_pandas = pd.read_csv(var_import_csv_file, sep="|", encoding = "utf8")

var_csv_from_pandas_path = knio.flow_variables['context.workflow.data-path'] + "from_pandas_csv_file.csv"
knio.flow_variables['var_csv_from_pandas_path']= var_csv_from_pandas_path

# , compression='gzip'
df_pandas.to_csv(var_csv_from_pandas_path, sep='|', encoding='utf-8')

knio.output_tables[0] = knio.Table.from_pyarrow(df_arrow)
knio.output_tables[1] = knio.Table.from_pandas(df_pandas)

External resources

• (48849) forum entry UTF-8 characters
• KNIME Python Integration Guide
• Meta Collection about KNIME and Python
• HUB: KNIME and UTF-8 characters with CSV between Python nodes with Pandas and Arrow