# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # distutils: language = c++ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_python cimport CTimePoint cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: ctypedef enum CFileType "arrow::fs::FileType": CFileType_NotFound "arrow::fs::FileType::NotFound" CFileType_Unknown "arrow::fs::FileType::Unknown" CFileType_File "arrow::fs::FileType::File" CFileType_Directory "arrow::fs::FileType::Directory" cdef cppclass CFileInfo "arrow::fs::FileInfo": CFileInfo() CFileInfo(CFileInfo&&) CFileInfo& operator=(CFileInfo&&) CFileInfo(const CFileInfo&) CFileInfo& operator=(const CFileInfo&) CFileType type() void set_type(CFileType type) c_string path() void set_path(const c_string& path) c_string base_name() int64_t size() void set_size(int64_t size) c_string extension() CTimePoint mtime() void set_mtime(CTimePoint mtime) cdef cppclass CFileSelector "arrow::fs::FileSelector": CFileSelector() c_string base_dir c_bool allow_not_found c_bool recursive cdef cppclass CFileLocator "arrow::fs::FileLocator": shared_ptr[CFileSystem] filesystem c_string path cdef cppclass CFileSystem "arrow::fs::FileSystem": shared_ptr[CFileSystem] shared_from_this() c_string type_name() const CResult[c_string] NormalizePath(c_string path) CResult[CFileInfo] GetFileInfo(const c_string& path) CResult[vector[CFileInfo]] GetFileInfo( const vector[c_string]& paths) CResult[vector[CFileInfo]] GetFileInfo(const CFileSelector& select) CStatus CreateDir(const c_string& path, c_bool recursive) CStatus DeleteDir(const c_string& path) CStatus DeleteDirContents(const c_string& path, c_bool missing_dir_ok) CStatus DeleteRootDirContents() CStatus DeleteFile(const c_string& path) CStatus DeleteFiles(const vector[c_string]& paths) CStatus Move(const c_string& src, const c_string& dest) CStatus CopyFile(const c_string& src, const c_string& dest) CResult[shared_ptr[CInputStream]] OpenInputStream( const c_string& path) CResult[shared_ptr[CRandomAccessFile]] OpenInputFile( const c_string& path) CResult[shared_ptr[COutputStream]] OpenOutputStream( const c_string& path, const shared_ptr[const CKeyValueMetadata]&) CResult[shared_ptr[COutputStream]] OpenAppendStream( const c_string& path, const shared_ptr[const CKeyValueMetadata]&) c_bool Equals(const CFileSystem& other) c_bool Equals(shared_ptr[CFileSystem] other) CResult[shared_ptr[CFileSystem]] CFileSystemFromUri \ "arrow::fs::FileSystemFromUri"(const c_string& uri, c_string* out_path) CResult[shared_ptr[CFileSystem]] CFileSystemFromUriOrPath \ "arrow::fs::FileSystemFromUriOrPath"(const c_string& uri, c_string* out_path) cdef cppclass CFileSystemGlobalOptions \ "arrow::fs::FileSystemGlobalOptions": c_string tls_ca_file_path c_string tls_ca_dir_path CStatus CFileSystemsInitialize "arrow::fs::Initialize" \ (const CFileSystemGlobalOptions& options) cdef cppclass CLocalFileSystemOptions "arrow::fs::LocalFileSystemOptions": c_bool use_mmap @staticmethod CLocalFileSystemOptions Defaults() c_bool Equals(const CLocalFileSystemOptions& other) cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): CLocalFileSystem() CLocalFileSystem(CLocalFileSystemOptions) CLocalFileSystemOptions options() cdef cppclass CSubTreeFileSystem \ "arrow::fs::SubTreeFileSystem"(CFileSystem): CSubTreeFileSystem(const c_string& base_path, shared_ptr[CFileSystem] base_fs) c_string base_path() shared_ptr[CFileSystem] base_fs() ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": CS3LogLevel log_level cdef cppclass CS3ProxyOptions "arrow::fs::S3ProxyOptions": c_string scheme c_string host int port c_string username c_string password c_bool Equals(const CS3ProxyOptions& other) @staticmethod CResult[CS3ProxyOptions] FromUriString "FromUri"( const c_string& uri_string) ctypedef enum CS3CredentialsKind "arrow::fs::S3CredentialsKind": CS3CredentialsKind_Anonymous "arrow::fs::S3CredentialsKind::Anonymous" CS3CredentialsKind_Default "arrow::fs::S3CredentialsKind::Default" CS3CredentialsKind_Explicit "arrow::fs::S3CredentialsKind::Explicit" CS3CredentialsKind_Role "arrow::fs::S3CredentialsKind::Role" CS3CredentialsKind_WebIdentity \ "arrow::fs::S3CredentialsKind::WebIdentity" cdef cppclass CS3Options "arrow::fs::S3Options": c_string region c_string endpoint_override c_string scheme c_bool background_writes shared_ptr[const CKeyValueMetadata] default_metadata c_string role_arn c_string session_name c_string external_id int load_frequency CS3ProxyOptions proxy_options CS3CredentialsKind credentials_kind void ConfigureDefaultCredentials() void ConfigureAccessKey(const c_string& access_key, const c_string& secret_key, const c_string& session_token) c_string GetAccessKey() c_string GetSecretKey() c_string GetSessionToken() c_bool Equals(const CS3Options& other) @staticmethod CS3Options Defaults() @staticmethod CS3Options Anonymous() @staticmethod CS3Options FromAccessKey(const c_string& access_key, const c_string& secret_key, const c_string& session_token) @staticmethod CS3Options FromAssumeRole(const c_string& role_arn, const c_string& session_name, const c_string& external_id, const int load_frequency) cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): @staticmethod CResult[shared_ptr[CS3FileSystem]] Make(const CS3Options& options) CS3Options options() c_string region() cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( const CS3GlobalOptions& options) cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() cdef CResult[c_string] ResolveS3BucketRegion(const c_string& bucket) cdef cppclass CHdfsOptions "arrow::fs::HdfsOptions": HdfsConnectionConfig connection_config int32_t buffer_size int16_t replication int64_t default_block_size @staticmethod CResult[CHdfsOptions] FromUriString "FromUri"( const c_string& uri_string) void ConfigureEndPoint(c_string host, int port) void ConfigureDriver(c_bool use_hdfs3) void ConfigureReplication(int16_t replication) void ConfigureUser(c_string user_name) void ConfigureBufferSize(int32_t buffer_size) void ConfigureBlockSize(int64_t default_block_size) void ConfigureKerberosTicketCachePath(c_string path) void ConfigureExtraConf(c_string key, c_string value) cdef cppclass CHadoopFileSystem "arrow::fs::HadoopFileSystem"(CFileSystem): @staticmethod CResult[shared_ptr[CHadoopFileSystem]] Make( const CHdfsOptions& options) CHdfsOptions options() cdef cppclass CMockFileSystem "arrow::fs::internal::MockFileSystem"( CFileSystem): CMockFileSystem(CTimePoint current_time) CStatus CCopyFiles "arrow::fs::CopyFiles"( const vector[CFileLocator]& sources, const vector[CFileLocator]& destinations, const CIOContext& io_context, int64_t chunk_size, c_bool use_threads) CStatus CCopyFilesWithSelector "arrow::fs::CopyFiles"( const shared_ptr[CFileSystem]& source_fs, const CFileSelector& source_sel, const shared_ptr[CFileSystem]& destination_fs, const c_string& destination_base_dir, const CIOContext& io_context, int64_t chunk_size, c_bool use_threads) # Callbacks for implementing Python filesystems # Use typedef to emulate syntax for std::function ctypedef void CallbackGetTypeName(object, c_string*) ctypedef c_bool CallbackEquals(object, const CFileSystem&) ctypedef void CallbackGetFileInfo(object, const c_string&, CFileInfo*) ctypedef void CallbackGetFileInfoVector(object, const vector[c_string]&, vector[CFileInfo]*) ctypedef void CallbackGetFileInfoSelector(object, const CFileSelector&, vector[CFileInfo]*) ctypedef void CallbackCreateDir(object, const c_string&, c_bool) ctypedef void CallbackDeleteDir(object, const c_string&) ctypedef void CallbackDeleteDirContents(object, const c_string&, c_bool) ctypedef void CallbackDeleteRootDirContents(object) ctypedef void CallbackDeleteFile(object, const c_string&) ctypedef void CallbackMove(object, const c_string&, const c_string&) ctypedef void CallbackCopyFile(object, const c_string&, const c_string&) ctypedef void CallbackOpenInputStream(object, const c_string&, shared_ptr[CInputStream]*) ctypedef void CallbackOpenInputFile(object, const c_string&, shared_ptr[CRandomAccessFile]*) ctypedef void CallbackOpenOutputStream( object, const c_string&, const shared_ptr[const CKeyValueMetadata]&, shared_ptr[COutputStream]*) ctypedef void CallbackNormalizePath(object, const c_string&, c_string*) cdef extern from "arrow/python/filesystem.h" namespace "arrow::py::fs" nogil: cdef cppclass CPyFileSystemVtable "arrow::py::fs::PyFileSystemVtable": PyFileSystemVtable() function[CallbackGetTypeName] get_type_name function[CallbackEquals] equals function[CallbackGetFileInfo] get_file_info function[CallbackGetFileInfoVector] get_file_info_vector function[CallbackGetFileInfoSelector] get_file_info_selector function[CallbackCreateDir] create_dir function[CallbackDeleteDir] delete_dir function[CallbackDeleteDirContents] delete_dir_contents function[CallbackDeleteRootDirContents] delete_root_dir_contents function[CallbackDeleteFile] delete_file function[CallbackMove] move function[CallbackCopyFile] copy_file function[CallbackOpenInputStream] open_input_stream function[CallbackOpenInputFile] open_input_file function[CallbackOpenOutputStream] open_output_stream function[CallbackOpenOutputStream] open_append_stream function[CallbackNormalizePath] normalize_path cdef cppclass CPyFileSystem "arrow::py::fs::PyFileSystem": @staticmethod shared_ptr[CPyFileSystem] Make(object handler, CPyFileSystemVtable vtable) PyObject* handler()