remove filecheck to enable symlinks (#7133)

* remove filecheck to enable symlinks * Update src/datasets/data_files.py * use relpath to resolve symlink --------- Co-authored-by: Quentin Lhoest <[email protected]>
huggingface · Dec 24, 2024 · 6c6a82a · 6c6a82a
1 parent 7298e5a
commit 6c6a82a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/datasets/data_files.py b/src/datasets/data_files.py
@@ -386,7 +386,7 @@ def resolve_pattern(
     matched_paths = [
         filepath if filepath.startswith(protocol_prefix) else protocol_prefix + filepath
         for filepath, info in fs.glob(pattern, detail=True, **glob_kwargs).items()
-        if info["type"] == "file"
+        if (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath))))
         and (xbasename(filepath) not in files_to_ignore)
         and not _is_inside_unrequested_special_dir(filepath, fs_pattern)
         and not _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern)