feat: support for multiple encodings in CSV files (#5756)

frontend/src/api/files.ts

@@ -3,14 +3,25 @@ import { useLayoutStore } from "@/stores/layout";
 import { baseURL } from "@/utils/constants";
 import { upload as postTus, useTus } from "./tus";
 import { createURL, fetchURL, removePrefix, StatusError } from "./utils";
+import { isEncodableResponse, makeRawResource } from "@/utils/encodings";
 
 export async function fetch(url: string, signal?: AbortSignal) {
+  const encoding = isEncodableResponse(url);
   url = removePrefix(url);
-  const res = await fetchURL(`/api/resources${url}`, { signal });
+  const res = await fetchURL(`/api/resources${url}`, {
+    signal,
+    headers: {
+      "X-Encoding": encoding ? "true" : "false",
+    },
+  });
 
   let data: Resource;
   try {
-    data = (await res.json()) as Resource;
+    if (res.headers.get("Content-Type") == "application/octet-stream") {
+      data = await makeRawResource(res, url);
+    } else {
+      data = (await res.json()) as Resource;
+    }
   } catch (e) {
     // Check if the error is an intentional cancellation
     if (e instanceof Error && e.name === "AbortError") {

frontend/src/components/files/CsvViewer.vue

@@ -4,35 +4,13 @@
       <i class="material-icons">error</i>
       <p>{{ displayError }}</p>
     </div>
-    <div v-else-if="data.headers.length === 0" class="csv-empty">
+    <div v-else-if="parsed.headers.length === 0" class="csv-empty">
       <i class="material-icons">description</i>
       <p>{{ $t("files.lonely") }}</p>
     </div>
     <div v-else class="csv-table-container" @wheel.stop @touchmove.stop>
-      <table class="csv-table">
-        <thead>
-          <tr>
-            <th v-for="(header, index) in data.headers" :key="index">
-              {{ header || `Column ${index + 1}` }}
-            </th>
-          </tr>
-        </thead>
-        <tbody>
-          <tr v-for="(row, rowIndex) in data.rows" :key="rowIndex">
-            <td v-for="(cell, cellIndex) in row" :key="cellIndex">
-              {{ cell }}
-            </td>
-          </tr>
-        </tbody>
-      </table>
-      <div class="csv-footer">
-        <div class="csv-info" v-if="data.rows.length > 100">
-          <i class="material-icons">info</i>
-          <span>
-            {{ $t("files.showingRows", { count: data.rows.length }) }}</span
-          >
-        </div>
-        <div class="column-separator">
+      <div class="csv-header">
+        <div class="header-select">
           <label for="columnSeparator">{{ $t("files.columnSeparator") }}</label>
           <select
             id="columnSeparator"
@@ -50,17 +28,61 @@
             </option>
           </select>
         </div>
+        <div class="header-select" v-if="isEncodedContent">
+          <label for="fileEncoding">{{ $t("files.fileEncoding") }}</label>
+          <select
+            id="fileEncoding"
+            class="input input--block"
+            v-model="selectedEncoding"
+          >
+            <option
+              v-for="encoding in availableEncodings"
+              :value="encoding"
+              :key="encoding"
+            >
+              {{ encoding }}
+            </option>
+          </select>
+        </div>
+      </div>
+      <table class="csv-table">
+        <thead>
+          <tr>
+            <th v-for="(header, index) in parsed.headers" :key="index">
+              {{ header || `Column ${index + 1}` }}
+            </th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr v-for="(row, rowIndex) in parsed.rows" :key="rowIndex">
+            <td v-for="(cell, cellIndex) in row" :key="cellIndex">
+              {{ cell }}
+            </td>
+          </tr>
+        </tbody>
+      </table>
+      <div class="csv-footer">
+        <div class="csv-info" v-if="parsed.rows.length > 100">
+          <i class="material-icons">info</i>
+          <span>
+            {{ $t("files.showingRows", { count: parsed.rows.length }) }}
+          </span>
+        </div>
       </div>
     </div>
   </div>
 </template>
 
 <script setup lang="ts">
-import { parseCSV, type CsvData } from "@/utils/csv";
-import { computed, ref } from "vue";
+import { computed, ref, watchEffect } from "vue";
+import { parse } from "csv-parse/browser/esm";
+import { useI18n } from "vue-i18n";
+import { availableEncodings, decode } from "@/utils/encodings";
+
+const { t } = useI18n({});
 
 interface Props {
-  content: string;
+  content: ArrayBuffer | string;
   error?: string;
 }
 
@@ -68,31 +90,43 @@ const props = withDefaults(defineProps<Props>(), {
   error: "",
 });
 
-const columnSeparator = ref([","]);
+const columnSeparator = ref([",", ";"]);
 
-const data = computed<CsvData>(() => {
-  try {
-    return parseCSV(props.content, columnSeparator.value);
-  } catch (e) {
-    console.error("Failed to parse CSV:", e);
-    return { headers: [], rows: [] };
-  }
+const selectedEncoding = ref("utf-8");
+
+const parsed = ref<CsvData>({ headers: [], rows: [] });
+
+const displayError = ref<string | null>(null);
+
+const isEncodedContent = computed(() => {
+  return props.content instanceof ArrayBuffer;
 });
 
-const displayError = computed(() => {
-  // External error takes priority (e.g., file too large)
-  if (props.error) {
-    return props.error;
+watchEffect(() => {
+  if (props.content !== "" && columnSeparator.value.length > 0) {
+    const content = isEncodedContent.value
+      ? decode(props.content as ArrayBuffer, selectedEncoding.value)
+      : props.content;
+    parse(
+      content as string,
+      { delimiter: columnSeparator.value, skip_empty_lines: true },
+      (error, output) => {
+        if (error) {
+          console.error("Failed to parse CSV:", error);
+          parsed.value = { headers: [], rows: [] };
+          displayError.value = t("files.csvLoadFailed", {
+            error: error.toString(),
+          });
+        } else {
+          parsed.value = {
+            headers: output[0],
+            rows: output.slice(1),
+          };
+          displayError.value = null;
+        }
+      }
+    );
   }
-  // Check for parse errors
-  if (
-    props.content &&
-    props.content.trim().length > 0 &&
-    data.value.headers.length === 0
-  ) {
-    return "Failed to parse CSV file";
-  }
-  return null;
 });
 </script>
 
@@ -213,10 +247,6 @@ const displayError = computed(() => {
   padding: 0.5rem;
 }
 
-.csv-footer > :only-child {
-  margin-left: auto;
-}
-
 .csv-info {
   display: flex;
   align-items: center;
@@ -230,18 +260,25 @@ const displayError = computed(() => {
   font-size: 0.875rem;
 }
 
-.column-separator {
+.csv-header {
+  display: flex;
+  justify-content: space-between;
+  padding: 0.25rem;
+}
+
+.header-select {
   display: flex;
   align-items: center;
   gap: 0.5rem;
+  margin-bottom: 0.5rem;
 }
 
-.column-separator > label {
+.header-select > label {
   font-size: small;
-  text-align: end;
+  max-width: 80px;
 }
 
-.column-separator > select {
+.header-select > select {
   margin-bottom: 0;
 }
 

frontend/src/i18n/en.json

@@ -88,7 +88,8 @@
       "comma": "Comma (,)",
       "semicolon": "Semicolon (;)",
       "both": "Both (,) and (;)"
-    }
+    },
+    "fileEncoding": "File Encoding"
   },
   "help": {
     "click": "select file or directory",

frontend/src/types/file.d.ts

@@ -21,6 +21,7 @@ interface Resource extends ResourceBase {
   index: number;
   subtitles?: string[];
   content?: string;
+  rawContent?: ArrayBuffer;
 }
 
 interface ResourceItem extends ResourceBase {
@@ -57,3 +58,8 @@ interface BreadCrumb {
   name: string;
   url: string;
 }
+
+interface CsvData {
+  headers: string[];
+  rows: string[][];
+}

frontend/src/utils/csv.ts

@@ -1,64 +0,0 @@
-export interface CsvData {
-  headers: string[];
-  rows: string[][];
-}
-
-/**
- * Parse CSV content into headers and rows
- * Supports quoted fields and handles commas within quotes
- */
-export function parseCSV(
-  content: string,
-  columnSeparator: Array<string>
-): CsvData {
-  if (!content || content.trim().length === 0) {
-    return { headers: [], rows: [] };
-  }
-
-  const lines = content.split(/\r?\n/);
-  const result: string[][] = [];
-
-  for (const line of lines) {
-    if (line.trim().length === 0) continue;
-
-    const row: string[] = [];
-    let currentField = "";
-    let inQuotes = false;
-
-    for (let i = 0; i < line.length; i++) {
-      const char = line[i];
-      const nextChar = line[i + 1];
-
-      if (char === '"') {
-        if (inQuotes && nextChar === '"') {
-          // Escaped quote
-          currentField += '"';
-          i++; // Skip next quote
-        } else {
-          // Toggle quote state
-          inQuotes = !inQuotes;
-        }
-      } else if (columnSeparator.includes(char) && !inQuotes) {
-        // Field separator
-        row.push(currentField);
-        currentField = "";
-      } else {
-        currentField += char;
-      }
-    }
-
-    // Add the last field
-    row.push(currentField);
-    result.push(row);
-  }
-
-  if (result.length === 0) {
-    return { headers: [], rows: [] };
-  }
-
-  // First row is headers
-  const headers = result[0];
-  const rows = result.slice(1);
-
-  return { headers, rows };
-}

frontend/src/utils/encodings.ts

@@ -0,0 +1,95 @@
+export const availableEncodings = [
+  "utf-8",
+  "ibm866",
+  "iso-8859-2",
+  "iso-8859-3",
+  "iso-8859-4",
+  "iso-8859-5",
+  "iso-8859-6",
+  "iso-8859-7",
+  "iso-8859-8",
+  "iso-8859-8-i",
+  "iso-8859-10",
+  "iso-8859-13",
+  "iso-8859-14",
+  "iso-8859-15",
+  "iso-8859-16",
+  "koi8-r",
+  "koi8-u",
+  "macintosh",
+  "windows-874",
+  "windows-1250",
+  "windows-1251",
+  "windows-1252",
+  "windows-1253",
+  "windows-1254",
+  "windows-1255",
+  "windows-1256",
+  "windows-1257",
+  "windows-1258",
+  "x-mac-cyrillic",
+  "gbk",
+  "gb18030",
+  "big5",
+  "euc-jp",
+  "iso-2022-jp",
+  "shift_jis",
+  "euc-kr",
+  "utf-16be",
+  "utf-16le",
+];
+
+export function decode(content: ArrayBuffer, encoding: string): string {
+  const decoder = new TextDecoder(encoding);
+  return decoder.decode(content);
+}
+
+export function isEncodableResponse(url: string): boolean {
+  const extensions = [".csv"];
+
+  if (typeof TextDecoder === "undefined") {
+    return false;
+  }
+
+  for (const extension of extensions) {
+    if (url.endsWith(extension)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export async function makeRawResource(
+  res: Response,
+  url: string
+): Promise<Resource> {
+  const buffer = await res.arrayBuffer();
+  return {
+    items: [],
+    numDirs: 0,
+    numFiles: 0,
+    sorting: {} as Sorting,
+    index: 0,
+    extension: getExtension(url),
+    isDir: false,
+    isSymlink: false,
+    path: url,
+    size: buffer.byteLength,
+    modified: new Date().toISOString(),
+    name: url.split("/").pop() || "",
+    type: "text",
+    mode: 0,
+    url: `/files${url}`,
+    rawContent: buffer,
+    content: decode(buffer, "utf-8"),
+  };
+}
+
+function getExtension(url: string): string {
+  const lastDotIndex = url.lastIndexOf(".");
+  if (lastDotIndex === -1) {
+    return "";
+  }
+  return url.substring(lastDotIndex);
+}

frontend/src/views/files/Editor.vue

@@ -171,25 +171,19 @@ onMounted(() => {
     `https://cdn.jsdelivr.net/npm/ace-builds@${ace_version}/src-min-noconflict/`
   );
 
-  editor.value = ace.edit("editor", {
-    value: fileContent,
-    showPrintMargin: false,
-    readOnly: fileStore.req?.type === "textImmutable",
-    theme: getEditorTheme(authStore.user?.aceEditorTheme ?? ""),
-    mode: modelist.getModeForPath(fileStore.req!.name).mode,
-    wrap: true,
-    enableBasicAutocompletion: true,
-    enableLiveAutocompletion: true,
-    enableSnippets: true,
-  });
-
-  editor.value.setFontSize(fontSize.value);
-  editor.value.focus();
-
-  editor.value.getSelection().on("changeSelection", () => {
-    isSelectionEmpty.value =
-      editor.value == null || editor.value.getSelectedText().length == 0;
-  });
+  if (!layoutStore.loading) {
+    initEditor(fileContent);
+  } else {
+    const unwatch = watchEffect(() => {
+      // Initialize editor when layout is loaded
+      if (!layoutStore.loading) {
+        setTimeout(() => {
+          initEditor(fileContent);
+          unwatch();
+        }, 50);
+      }
+    });
+  }
 });
 
 onBeforeUnmount(() => {
@@ -218,6 +212,23 @@ onBeforeRouteUpdate((to, from, next) => {
   });
 });
 
+const initEditor = (fileContent: string) => {
+  editor.value = ace.edit("editor", {
+    value: fileContent,
+    showPrintMargin: false,
+    readOnly: fileStore.req?.type === "textImmutable",
+    theme: getEditorTheme(authStore.user?.aceEditorTheme ?? ""),
+    mode: modelist.getModeForPath(fileStore.req!.name).mode,
+    wrap: true,
+    enableBasicAutocompletion: true,
+    enableLiveAutocompletion: true,
+    enableSnippets: true,
+  });
+
+  editor.value.setFontSize(fontSize.value);
+  editor.value.focus();
+};
+
 const keyEvent = (event: KeyboardEvent) => {
   if (event.code === "Escape") {
     close();

frontend/src/views/files/Preview.vue

@@ -253,7 +253,7 @@ const hoverNav = ref<boolean>(false);
 const autoPlay = ref<boolean>(false);
 const previousRaw = ref<string>("");
 const nextRaw = ref<string>("");
-const csvContent = ref<string>("");
+const csvContent = ref<ArrayBuffer | string>("");
 const csvError = ref<string>("");
 
 const player = ref<HTMLVideoElement | HTMLAudioElement | null>(null);
@@ -393,7 +393,11 @@ const updatePreview = async () => {
     if (fileStore.req.size > CSV_MAX_SIZE) {
       csvError.value = t("files.csvTooLarge");
     } else {
-      csvContent.value = fileStore.req.content ?? "";
+      if (fileStore.req.rawContent != null) {
+        csvContent.value = fileStore.req.rawContent;
+      } else {
+        csvContent.value = fileStore.req.content ?? "";
+      }
     }
   }