Source code for

# Copyright 1999-2021 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
from typing import Any, Callable, List

from .... import oscar as mo
from import StorageLevel
from ....utils import serialize_serializable, deserialize_serializable
from ...web import web_api, MarsServiceWebAPIHandler, MarsWebAPIClientMixin
from ..core import DataInfo
from .core import AbstractStorageAPI

class StorageWebAPIHandler(MarsServiceWebAPIHandler):
    _root_pattern = "/api/session/(?P<session_id>[^/]+)/storage"

    async def _get_oscar_meta_api(self, session_id: str):
        from ...meta import MetaAPI

        return await self._get_api_by_key(MetaAPI, session_id)

    async def _get_storage_api_by_object_id(self, session_id: str, object_id: str):
        from .oscar import StorageAPI

        meta_api = await self._get_oscar_meta_api(session_id)
        bands = (await meta_api.get_chunk_meta(object_id, ["bands"])).get("bands")
        if not bands:
            raise KeyError
        return await StorageAPI.create(session_id, bands[0][0], bands[0][1])

    @web_api("(?P<data_key>[^/]+)", method="get")
    async def get_data(self, session_id: str, data_key: str):
        oscar_api = await self._get_storage_api_by_object_id(session_id, data_key)
        result = await oscar_api.get(data_key)

    @web_api("batch/get", method="post")
    async def get_batch_data(self, session_id: str):
        body_args = deserialize_serializable(self.request.body)
        storage_api_to_gets = defaultdict(list)
        storage_api_to_idx = defaultdict(list)
        results = [None] * len(body_args)
        for i, (data_key, conditions, error) in enumerate(body_args):
            oscar_api = await self._get_storage_api_by_object_id(session_id, data_key)
                oscar_api.get.delay(data_key, conditions=conditions, error=error)
        for api, fetches in storage_api_to_gets.items():
            data_list = await api.get.batch(*fetches)
            for idx, data in zip(storage_api_to_idx[api], data_list):
                results[idx] = data
        res_data = serialize_serializable(results)

    @web_api("(?P<data_key>[^/]+)", method="post")
    async def get_data_by_post(self, session_id: str, data_key: str):
        body_args = (
            deserialize_serializable(self.request.body) if self.request.body else None
        conditions = body_args.get("conditions")

        oscar_api = await self._get_storage_api_by_object_id(session_id, data_key)
        result = await oscar_api.get(data_key, conditions)

    @web_api("(?P<data_key>[^/]+)", method="put")
    async def put_data(self, session_id: str, data_key: str):
        level = self.get_argument("level", None) or "MEMORY"
        level = getattr(StorageLevel, level.upper())

        oscar_api = await self._get_storage_api_by_object_id(session_id, data_key)
        res = await oscar_api.put(
            data_key, deserialize_serializable(self.request.body), level

    @web_api("(?P<data_key>[^/]+)", method="get", arg_filter={"action": "get_infos"})
    async def get_infos(self, session_id: str, data_key: str):
        oscar_api = await self._get_storage_api_by_object_id(session_id, data_key)
        res = await oscar_api.get_infos(data_key)

web_handlers = {StorageWebAPIHandler.get_root_pattern(): StorageWebAPIHandler}

[docs]class WebStorageAPI(AbstractStorageAPI, MarsWebAPIClientMixin):
[docs] def __init__( self, session_id: str, address: str, band_name: str, request_rewriter: Callable = None, ): self._session_id = session_id self._address = address.rstrip("/") self._band_name = band_name self.request_rewriter = request_rewriter
@mo.extensible async def get( self, data_key: str, conditions: List = None, error: str = "raise" ) -> Any: path = f"{self._address}/api/session/{self._session_id}/storage/{data_key}" params = dict(error=error) if conditions is not None: params["conditions"] = conditions body = serialize_serializable(params) res = await self._request_url( path=path, method="POST", headers={"Content-Type": "application/octet-stream"}, data=body, ) return deserialize_serializable(res.body) @get.batch async def get_batch(self, args_list, kwargs_list): get_chunks = [] for args, kwargs in zip(args_list, kwargs_list): data_key, conditions, error = self.get.bind(*args, **kwargs) get_chunks.append([data_key, conditions, error]) path = f"{self._address}/api/session/{self._session_id}/storage/batch/get" res = await self._request_url( path=path, method="POST", data=serialize_serializable(get_chunks), ) return deserialize_serializable(res.body) @mo.extensible async def put( self, data_key: str, obj: object, level: StorageLevel = StorageLevel.MEMORY ) -> DataInfo: params = dict( path = f"{self._address}/api/session/{self._session_id}/storage/{data_key}" res = await self._request_url( path=path, method="PUT", params=params, headers={"Content-Type": "application/octet-stream"}, data=serialize_serializable(obj), ) return deserialize_serializable(res.body) @mo.extensible async def get_infos(self, data_key: str) -> List[DataInfo]: path = f"{self._address}/api/session/{self._session_id}/storage/{data_key}" params = dict(action="get_infos") res = await self._request_url( path=path, method="GET", headers={"Content-Type": "application/octet-stream"}, params=params, ) return deserialize_serializable(res.body)