pinecone_datasets.fs 

 1from typing import Union
 2
 3import gcsfs
 4import s3fs
 5from fsspec.implementations.local import LocalFileSystem
 6
 7from pinecone_datasets import cfg
 8
 9
10def get_cloud_fs(
11    path, **kwargs
12) -> Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem, LocalFileSystem]:
13    """
14    returns a filesystem object for the given path, if it is a cloud storage path (gs:// or s3://)
15
16    Args:
17        path (str): the path to the file or directory
18        **kwargs: additional arguments to pass to the filesystem constructor
19
20    Returns:
21        fs: Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem] - the filesystem object
22    """
23    is_anon = path == cfg.Storage.endpoint
24    if path.startswith("gs://") or "storage.googleapis.com" in path:
25        fs = gcsfs.GCSFileSystem(token="anon" if is_anon else None, **kwargs)
26    elif path.startswith("s3://") or "s3.amazonaws.com" in path:
27        fs = s3fs.S3FileSystem(anon=is_anon, **kwargs)
28    else:
29        fs = LocalFileSystem()
30    return fs
def get_cloud_fs( path, **kwargs) -> Union[gcsfs.core.GCSFileSystem, s3fs.core.S3FileSystem, fsspec.implementations.local.LocalFileSystem]: 
11def get_cloud_fs(
12    path, **kwargs
13) -> Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem, LocalFileSystem]:
14    """
15    returns a filesystem object for the given path, if it is a cloud storage path (gs:// or s3://)
16
17    Args:
18        path (str): the path to the file or directory
19        **kwargs: additional arguments to pass to the filesystem constructor
20
21    Returns:
22        fs: Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem] - the filesystem object
23    """
24    is_anon = path == cfg.Storage.endpoint
25    if path.startswith("gs://") or "storage.googleapis.com" in path:
26        fs = gcsfs.GCSFileSystem(token="anon" if is_anon else None, **kwargs)
27    elif path.startswith("s3://") or "s3.amazonaws.com" in path:
28        fs = s3fs.S3FileSystem(anon=is_anon, **kwargs)
29    else:
30        fs = LocalFileSystem()
31    return fs

returns a filesystem object for the given path, if it is a cloud storage path (gs:// or s3://)

Arguments:
  • path (str): the path to the file or directory
  • **kwargs: additional arguments to pass to the filesystem constructor
Returns:

fs: Union[gcsfs.GCSFileSystem, s3fs.S3FileSystem] - the filesystem object