benten download

目的

  • 複数ファイルのダウンロードを行います。

Warning

事前にログイン認証を行う必要があります。

概要

  • データ検索条件にマッチした複数のファイルをダウンロードします。
  • ダウンロードできるファイルは認証時のアカウントでアクセスが許可されたファイルのみ対象となります。
  • ダウンロード後はハッシュのチェックを行い、データに矛盾がないことを確認します。

利用例

  • Help
$ benten.py download -h
usage: benten download [-h] [--register_name [REGISTER_NAME]] [--file [FILE]]
                       [--flag_recursive] [--flag_own] [--flag_not_download]
                       [--out_directory [OUT_DIRECTORY]] [--disable_hash]

optional arguments:
  -h, --help            show this help message and exit
  --register_name [REGISTER_NAME], -r [REGISTER_NAME]
                        register_name
  --file [FILE], -f [FILE]
                        file or directory
  --flag_recursive, -s  download file_list with recursive search for directory
  --flag_own, -o        output with own file_list
  --flag_not_download, -n
                        not download file if the file is available
  --out_directory [OUT_DIRECTORY], -d [OUT_DIRECTORY]
                        directory for output file if the file is available
  --disable_hash        disable hash check in downloaded file
  • ディレクトリを指定してダウンロード
$ benten.py download -f "/SPring-8/BL14B2/data2/temp"
### benten download ###
[Repository::authorize] access_token = 579773811dc7401ea1307c99d2a88a82
 ==> file_list
{
    "total_count":1,
    "size":0,
    "begin_from":0,
    "file_list":[
        {
            "size":109,
            "name":"/SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613.json",
            "time":"2019-03-12 11:33:02",
            "hash":"5feb764597a9de85748a874897e567b7",
            "uuid_name":"d82d86d1-9c91-4cd0-b9a4-16f489803d9b",
            "register_name":"/SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613"
        },
        {
            "size":458,
            "name":"/SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613.system.json",
            "time":"2019-03-09 21:17:47",
            "hash":"f5c1e6361041f0a776bc7fcc1d5559ad",
            "uuid_name":"d969744f-bb40-4c58-8056-3663b2484abd",
            "register_name":"/SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613"
        }
    ]
}
==> downloaded with file = ./SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613.json
==> check consitency with hash = 5feb764597a9de85748a874897e567b7
==> downloaded with file = ./SPring-8/BL14B2/data2/temp/Cu-K_Cu-foil_Si111_50ms_120613.system.json
==> check consitency with hash = f5c1e6361041f0a776bc7fcc1d5559ad
==> elaspted time = 0 sec

Pythonモジュールとの対応

#!/usr/bin/env python
'''
  example: download

  usage: python benten_download.py ...

'''

import benten_client

import time

import argparse
from logging import getLogger, StreamHandler, DEBUG
logger = getLogger(__name__)
handler = StreamHandler()
handler.setLevel(DEBUG)
logger.setLevel(DEBUG)
logger.addHandler(handler)
logger.propagate = False

# ... parameters
parser = argparse.ArgumentParser(description="example: download")
parser.add_argument("--register_name", "-r", nargs="?",
                    action="append", help="register_name")
parser.add_argument("--file", "-f", nargs="?",
                    action="append", help="file or directory")
parser.add_argument("--flag_recursive", "-s", action="store_const", const=1,
                    help="download file_list with recursive search for directory")
parser.add_argument("--flag_own", "-o", action="store_const",
                    const=1, help="output with own file_list")
parser.add_argument("--flag_not_download", "-n", action="store_const",
                    const=1, help="not download file if the file is available")
parser.add_argument("--out_directory", "-d", nargs="?",
                    help="directory for output file if the file is available")
parser.add_argument("--disable_hash", action="store_const",
                    const=1, help="disable hash check in downloaded file")

args = parser.parse_args()

register_name_list = args.register_name
file_list = args.file

flag_recursive = args.flag_recursive
flag_own = args.flag_own

flag_not_download = args.flag_not_download

out_directory = args.out_directory
disable_hash = args.disable_hash

if register_name_list is None and file_list is None:
    raise Exception("Need input for register_name or file")

t_begin = time.time()

v = {}
if register_name_list is not None:
    v["register_name_list"] = register_name_list
if file_list is not None:
    v["file_list"] = file_list
if flag_recursive is not None:
    v["flag_recursive"] = flag_recursive
if flag_own is not None:
    v["flag_own"] = flag_own

repo = benten_client.Repository() 

logger.debug("# set access token")

repo.authorize(benten_client.access_token())

logger.debug("# get file list for download")

ret_dict = repo.data.files.post(**v)
benten_client.log(" ==> file_list")
benten_client.out_json(ret_dict)

if flag_not_download not in [1, True]:

    file_list = ret_dict.get("file_list",[])

    flag_path = True
    for f in file_list:

        v = {}
        v["file"] = f["name"]
        ret_dict = repo.download.file.post(
            v,out_directory=out_directory, flag_path=flag_path, 
            disable_hash=disable_hash)
        if ret_dict is not None:
            benten_client.log("==> response")
            benten_client.out_json(ret_dict)

    t_end = time.time()
    dt = int(t_end - t_begin)
    benten_client.log("==> elaspted time = {} sec".format(dt))