Class which manages downloads from a URL or GitHub and download/upload with the Girder database by using libcurl. More...

Collaboration diagram for Feel::RemoteData:

Classes
class	CKAN

class	ContentsInfo
	Content info data structure. More...

struct	FileInfo

struct	FolderInfo

class	Girder

class	Github

struct	ItemInfo

class	URL

Public Member Functions
bool	canDownload () const
	Return true if enough information is available to download a file.

bool	canUpload () const
	Return true if enough information is available to upload data.

ContentsInfo	contents () const
	Get contents of remote data (folder, item, file) More...

ContentsInfo	contents (RemoteDataProgress &progress) const
	Get contents of remote data (folder, item, file) with progress reporting. More...

nl::json	createDataset (const std::string &datasetName) const

std::vector< std::pair< std::string, std::string > >	createFolder (std::string const &folderPath, std::string const &parentId="", bool sync=true) const
	Create folders hierarchy on remote storage. More...

std::vector< std::pair< std::string, std::string > >	createItem (std::string const &itemPath, std::string const &parentId, bool sync=true) const
	Create item on remote storage. More...

bool	deleteDataset (const std::string &datasetId) const

bool	deleteResource (const nl::json &resourceId, const std::string &token="") const
	Delete a resource by id. More...

std::vector< std::string >	download (std::string const &dir, std::string const &filename, int timeout) const
	Download the file with timeout. More...

std::vector< std::string >	download (std::string const &dir=Environment::downloadsRepository(), std::string const &filename="") const
	Download the file. More...

std::vector< std::string >	listOrganizations () const
	List organizations available on the remote data platform (CKAN only) More...

	RemoteData (RemoteData &&)=default

	RemoteData (RemoteData const &)=default

FEELPP_DEPRECATED	RemoteData (std::string const &desc, WorldComm const &worldComm)

	RemoteData (std::string const &desc, worldcomm_ptr_t const &worldComm=Environment::worldCommPtr())

void	replaceFile (std::string const &filePath, std::string const &fileId) const
	Replace contents of a file. More...

void	replaceFile (std::vector< std::pair< std::string, std::string > > const &filesToReplace) const
	Replace contents of files. More...

nl::json	resourceLookup (const std::string &path, const std::string &token="") const
	Lookup a resource by path. More...

void	setWorldComm (WorldComm &wc)
	Set the WorldComm.

std::vector< std::string >	upload (std::string const &dataPath, std::string const &parentId, bool sync, int timeout) const
	Upload data on a remote storage with timeout. More...

std::vector< std::string >	upload (std::string const &dataPath, std::string const &parentId="", bool sync=true) const
	Upload data on a remote storage. More...

std::vector< std::vector< std::string > >	upload (std::vector< std::pair< std::string, std::string > > const &dataToUpload, bool sync=true) const
	Upload data on Girder. More...

WorldComm &	worldComm () const
	Return world comm.

Detailed Description

Class which manages downloads from a URL or GitHub and download/upload with the Girder database by using libcurl.

Member Function Documentation

◆ contents() [1/2]

ContentsInfo Feel::RemoteData::contents ( ) const

Get contents of remote data (folder, item, file)

Returns: : (Folders info, Items info, Files info)

Referenced by Feel::RemoteData::Girder::contents().

◆ contents() [2/2]

ContentsInfo Feel::RemoteData::contents ( RemoteDataProgress & progress ) const

Get contents of remote data (folder, item, file) with progress reporting.

Parameters

progress : progress reporter for debug output

Returns: : (Folders info, Items info, Files info)

◆ createFolder()

std::vector< std::pair< std::string, std::string > > Feel::RemoteData::createFolder	(	std::string const &	folderPath,
		std::string const &	parentId = `""`,
		bool	sync = `true`
	)		const

Create folders hierarchy on remote storage.

Parameters

folderPath	: the folder hierarchy
parentId	: id where folder is created, empty means to use folder id in the desc
sync	: apply MPI synchronization with returned infos (else only master rank has these infos)

Returns: : vector of (subdir name, subdir id)

◆ createItem()

std::vector< std::pair< std::string, std::string > > Feel::RemoteData::createItem	(	std::string const &	itemPath,
		std::string const &	parentId,
		bool	sync = `true`
	)		const

Create item on remote storage.

Parameters

itemPath	: the item
parentId	: id where item is created, empty means to use folder id in the desc
sync	: apply MPI synchronization with returned infos (else only master rank has these infos)

◆ deleteResource()

bool Feel::RemoteData::deleteResource	(	const nl::json &	resourceId,
		const std::string &	token = `""`
	)		const

Delete a resource by id.

Parameters

resourceId	: the id of the resource in Girder
token	: authentication token

Returns: : true if the resource was deleted

◆ download() [1/2]

std::vector< std::string > Feel::RemoteData::download	(	std::string const &	dir,
		std::string const &	filename,
		int	timeout
	)		const

Download the file with timeout.

Parameters

dir	: the directory where the file is downloaded
filename	: the filename of the downloaded file
timeout	: timeout in milliseconds for HTTP requests (default: 30000)

Returns: : the path of the downloaded file

◆ download() [2/2]

std::vector< std::string > Feel::RemoteData::download	(	std::string const &	dir = `Environment::downloadsRepository()`,
		std::string const &	filename = `""`
	)		const

Download the file.

Parameters

dir	: the directory where the file is downloaded
filename	: the filename of the downloaded file

Returns: : the path of the downloaded file

Referenced by Feel::RemoteData::Girder::download().

◆ listOrganizations()

std::vector< std::string > Feel::RemoteData::listOrganizations ( ) const

List organizations available on the remote data platform (CKAN only)

Returns: : vector of organization names

◆ replaceFile() [1/2]

void Feel::RemoteData::replaceFile	(	std::string const &	filePath,
		std::string const &	fileId
	)		const

Replace contents of a file.

Parameters

filePath	: path of new file
fileId	: id of the file to replace

Referenced by Feel::RemoteData::Girder::replaceFile().

◆ replaceFile() [2/2]

void Feel::RemoteData::replaceFile ( std::vector< std::pair< std::string, std::string > > const & filesToReplace ) const

Replace contents of files.

Parameters

filesToReplace : vector of (path of new file, file id to replace)

◆ resourceLookup()

nl::json Feel::RemoteData::resourceLookup	(	const std::string &	path,
		const std::string &	token = `""`
	)		const

Lookup a resource by path.

Parameters

path	: the path to the resource
token	: authentication token

Returns: : JSON object containing resource information

◆ upload() [1/3]

std::vector< std::string > Feel::RemoteData::upload	(	std::string const &	dataPath,
		std::string const &	parentId,
		bool	sync,
		int	timeout
	)		const

Upload data on a remote storage with timeout.

Parameters

dataPath	: a path of a file or a folder
parentId	: id where folder is created, empty means to use folder id in the desc
sync	: apply MPI synchronization with returned infos (else only master rank has these infos)
timeout	: timeout in milliseconds for HTTP requests (default: 30000)

Returns: : vector of paths of the uploaded files or folders

◆ upload() [2/3]

std::vector< std::string > Feel::RemoteData::upload	(	std::string const &	dataPath,
		std::string const &	parentId = `""`,
		bool	sync = `true`
	)		const

Upload data on a remote storage.

Parameters

dataPath	: a path of a file or a folder
parentId	: id where folder is created, empty means to use folder id in the desc
sync	: apply MPI synchronization with returned infos (else only master rank has these infos)

Returns: : vector of paths of the uploaded files or folders

Referenced by Feel::RemoteData::CKAN::upload(), and Feel::RemoteData::Girder::upload().

◆ upload() [3/3]

std::vector< std::vector< std::string > > Feel::RemoteData::upload	(	std::vector< std::pair< std::string, std::string > > const &	dataToUpload,
		bool	sync = `true`
	)		const

Upload data on Girder.

Parameters

dataToUpload	: vector of (paths of a file or a folder, ids where folder is created, empty means to use folder id in the desc)
sync	: apply MPI synchronization with returned infos (else only master rank has these infos)

Returns: : vector of vectors of file ids uploaded

Classes

Public Member Functions

Detailed Description

Member Function Documentation

◆ contents() [1/2]

◆ contents() [2/2]

◆ createFolder()

◆ createItem()

◆ deleteResource()

◆ download() [1/2]

◆ download() [2/2]

◆ listOrganizations()

◆ replaceFile() [1/2]

◆ replaceFile() [2/2]

◆ resourceLookup()

◆ upload() [1/3]

◆ upload() [2/3]

◆ upload() [3/3]