#!/usr/bin/env python

#

# Copyright 2007 Google LLC

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""A Python blobstore API used by app developers.

This module contains methods that are used to interface with Blobstore API.

The module includes a `db.Model`-like class that represents a reference to a

very large blob. The module imports a `db.Key`-like class that represents a blob

key.

"""

import base64

import cgi

import collections

import email

import email.message

import http

import re

from google.appengine.api import datastore

from google.appengine.api import datastore_errors

from google.appengine.api import datastore_types

from google.appengine.api.blobstore import blobstore

from google.appengine.ext import db

import six

from six.moves import urllib

__all__ = ['BLOB_INFO_KIND',

'BLOB_KEY_HEADER',

'BLOB_MIGRATION_KIND',

'BLOB_RANGE_HEADER',

'BlobFetchSizeTooLargeError',

'BlobInfo',

'BlobInfoParseError',

'BlobKey',

'BlobMigrationRecord',

'BlobNotFoundError',

'BlobReferenceProperty',

'BlobReader',

'FileInfo',

'FileInfoParseError',

'DataIndexOutOfRangeError',

'PermissionDeniedError',

'Error',

'InternalError',

'MAX_BLOB_FETCH_SIZE',

'UPLOAD_INFO_CREATION_HEADER',

'CLOUD_STORAGE_OBJECT_HEADER',

'create_rpc',

'create_upload_url',

'create_upload_url_async',

'delete',

'delete_async',

'fetch_data',

'fetch_data_async',

'create_gs_key',

'create_gs_key_async',

'GS_PREFIX',

'get',

'parse_blob_info',

'parse_file_info',

'RangeFormatError',

'UnsupportedRangeFormatError',

'BlobstoreDownloadHandler',

'BlobstoreUploadHandler',]

Error = blobstore.Error

InternalError = blobstore.InternalError

BlobFetchSizeTooLargeError = blobstore.BlobFetchSizeTooLargeError

BlobNotFoundError = blobstore.BlobNotFoundError

_CreationFormatError = blobstore._CreationFormatError

DataIndexOutOfRangeError = blobstore.DataIndexOutOfRangeError

PermissionDeniedError = blobstore.PermissionDeniedError

BlobKey = blobstore.BlobKey

create_rpc = blobstore.create_rpc

create_upload_url = blobstore.create_upload_url

create_upload_url_async = blobstore.create_upload_url_async

delete = blobstore.delete

delete_async = blobstore.delete_async

create_gs_key = blobstore.create_gs_key

create_gs_key_async = blobstore.create_gs_key_async

BLOB_INFO_KIND = blobstore.BLOB_INFO_KIND

BLOB_MIGRATION_KIND = blobstore.BLOB_MIGRATION_KIND

BLOB_KEY_HEADER = blobstore.BLOB_KEY_HEADER

BLOB_RANGE_HEADER = blobstore.BLOB_RANGE_HEADER

MAX_BLOB_FETCH_SIZE = blobstore.MAX_BLOB_FETCH_SIZE

UPLOAD_INFO_CREATION_HEADER = blobstore.UPLOAD_INFO_CREATION_HEADER

CLOUD_STORAGE_OBJECT_HEADER = blobstore.CLOUD_STORAGE_OBJECT_HEADER

GS_PREFIX = blobstore.GS_PREFIX

_CONTENT_DISPOSITION_FORMAT = b'attachment; filename="%s"'

_CONTENT_DISPOSITION_FORMAT_UTF8 = (b'attachment; filename="%s"; '

b'filename*=utf-8\'\'%s')

_SEND_BLOB_PARAMETERS = frozenset(['use_range'])

_RANGE_NUMERIC_FORMAT = r'([0-9]*)-([0-9]*)'

_RANGE_FORMAT = r'([a-zA-Z]+)=%s' % _RANGE_NUMERIC_FORMAT

_RANGE_FORMAT_REGEX = re.compile('^%s$' % _RANGE_FORMAT)

_UNSUPPORTED_RANGE_FORMAT_REGEX = re.compile(

'^%s(?:,%s)+$' % (_RANGE_FORMAT, _RANGE_NUMERIC_FORMAT))

_BYTES_UNIT = 'bytes'

class BlobInfoParseError(Error):

"""The CGI parameter does not contain a valid `BlobInfo` record."""

class FileInfoParseError(Error):

"""The CGI parameter does not contain a valid `FileInfo` record."""

class RangeFormatError(Error):

"""Raised when Range header incorrectly formatted."""

class UnsupportedRangeFormatError(RangeFormatError):

"""Raised when Range format is correct, but not supported."""

class _GqlQuery(db.GqlQuery):

"""GqlQuery class that explicitly sets `model-class`.

This does the same as the original `db.GqlQuery` class except that it does

not try to find the model class based on the compiled GQL query. The

caller instead provides the query with a model class to use for construction.

Note:

This class is required for compatibility with the current `db.py` query

mechanism but will be removed in the future. DO NOT USE.

"""

def __init__(self, query_string, model_class, *args, **kwds):

"""Constructor.

Args:

query_string: A properly formatted GQL query string.

model_class: The model class from which entities are constructed.

*args: Positional arguments that are used to bind numeric references in

the query.

**kwds: Dictionary-based arguments for named references.

"""

from google.appengine.ext import gql

app = kwds.pop('_app', None)

self._proto_query = gql.GQL(query_string, _app=app, namespace='')

super(db.GqlQuery, self).__init__(model_class)

self.bind(*args, **kwds)

class BlobInfo(object):

"""Information about blobs in Blobstore.

This is a `db.Model`-like class that contains information about blobs that are

stored by an application. Like `db.Model`, this class is backed by a

Datastore entity; however, `BlobInfo` instances are read-only and have a much

more limited interface.

Each `BlobInfo` has a key of type `BlobKey` associated with it. This key is

specific to the Blobstore API and is not compatible with `db.get`. The key

can be used for quick lookup by passing it to `BlobInfo.get`. This key

converts easily to a string, which is web safe and can be embedded in URLs.

Properties:

- content_type: The content type of the blob.

- creation: The creation date of the blob, or when it was uploaded.

- filename: The file name that the user selected from their machine.

- size: The size of the uncompressed blob.

- md5_hash: The MD5 hash value of the uploaded blob.

gs_object_name: The name of the object, if the blob is stored in

Google Cloud Storage, in the form /[bucket-name]/[object-name]

All properties are read-only. Attempting to assign a value to a property

will raise a `NotImplementedError`.

"""

_unindexed_properties = frozenset([])

_all_properties = frozenset(['content_type', 'creation', 'filename',

'size', 'md5_hash', 'gs_object_name'])

@property

def content_type(self):

"""Returns the content type of the blob.

Returns:

The content type of the blob.

"""

return self.__get_value('content_type')

@property

def creation(self):

"""Returns the creation date or time of upload of the blob.

Returns:

The creation date or time of upload of the blob.

"""

return self.__get_value('creation')

@property

def filename(self):

"""Returns the file name that the user selected from their machine.

Returns:

The file name that the user selected.

"""

return self.__get_value('filename')

@property

def size(self):

"""Returns the size of the uncompressed blob.

Returns:

The size of the uncompressed blob.

"""

return self.__get_value('size')

@property

def md5_hash(self):

"""Returns the MD5 hash value of the uncompressed blob.

Returns:

The hash value of the uncompressed blob.

"""

return self.__get_value('md5_hash')

@property

def gs_object_name(self):

return self.__get_value('gs_object_name')

def __init__(self, entity_or_blob_key, _values=None):

"""Constructor for wrapping the blobstore entity.

The constructor should not be used outside of this package and tests.

Args:

entity_or_blob_key: The datastore entity or blob key that represents the

blob reference.

_values: Optional; not recommended. This argument passes the associated

entity when `entity_or_blob_key` is a blob key.

"""

if isinstance(entity_or_blob_key, datastore.Entity):

self.__entity = entity_or_blob_key

self.__key = BlobKey(entity_or_blob_key.key().name())

elif isinstance(entity_or_blob_key, BlobKey):

self.__entity = _values

self.__key = entity_or_blob_key

else:

raise TypeError('Must provide Entity or BlobKey')

@classmethod

def from_entity(cls, entity):

"""Converts an entity to `BlobInfo`.

Note:

This method is required for compatibility with the current `db.py` query

mechanism but will be removed in the future. DO NOT USE.

Args:

entity: The entity that you are trying to convert.

Returns:

The `BlobInfo` that was converted from the entity.

"""

return BlobInfo(entity)

@classmethod

def properties(cls):

"""Defines the set of properties that belong to `BlobInfo`.

Note:

This method is required for compatibility with the current `db.py` query

mechanism but will be removed in the future. DO NOT USE.

Returns:

A set of all of the properties that belong to `BlobInfo`.

"""

return set(cls._all_properties)

def __get_value(self, name):

"""Gets a `BlobInfo` value, loading an entity if necessary.

This method allows lazy loading of the underlying datastore entity. It

should never be invoked directly.

Args:

name: The name of property for which you want to retrieve the value.

Returns:

The value of the `BlobInfo` property from the entity.

"""

if self.__entity is None:

self.__entity = datastore.Get(

datastore_types.Key.from_path(

self.kind(), str(self.__key), namespace=''))

return self.__entity.get(name, None)

def key(self):

"""Gets the key for a blob.

Returns:

The `BlobKey` instance that identifies this blob.

"""

return self.__key

def delete(self, _token=None):

"""Permanently deletes a blob from Blobstore."""

delete(self.key(), _token=_token)

def open(self, *args, **kwargs):

"""Returns a `BlobReader` for this blob.

Args:

*args: Arguments to pass to the `BlobReader` constructor.

**kwargs: Keyword arguments to pass to the `BlobReader` constructor.

Returns:

A `BlobReader` instance.

"""

return BlobReader(self, *args, **kwargs)

@classmethod

def get(cls, blob_keys):

"""Retrieves a `BlobInfo` by key or by a list of keys.

Args:

blob_keys: A key or a list of keys. Keys can be in string, Unicode, or

`BlobKey` format.

Returns:

A `BlobInfo` instance that is associated with the provided key or a list

of `BlobInfo` instances if a list of keys was provided. Keys that are not

found in Blobstore return `None`.

"""

blob_keys = cls.__normalize_and_convert_keys(blob_keys)

try:

entities = datastore.Get(blob_keys)

except datastore_errors.EntityNotFoundError:

return None

if isinstance(entities, datastore.Entity):

return BlobInfo(entities)

else:

references = []

for entity in entities:

if entity is not None:

references.append(BlobInfo(entity))

else:

references.append(None)

return references

@classmethod

def all(cls):

"""Creates a query for all `BlobInfo` objects associated with the app.

Returns:

A `db.Query` object that queries over `BlobInfo`'s datastore kind.

"""

return db.Query(model_class=cls, namespace='')

@classmethod

def __factory_for_kind(cls, kind):

if kind == BLOB_INFO_KIND:

return BlobInfo

raise ValueError('Cannot query for kind %s' % kind)

@classmethod

def gql(cls, query_string, *args, **kwds):

"""Returns a query using a GQL query string.

See the `GQL source` for more information about GQL.

Args:

query_string: A properly formatted GQL query string that omits

`SELECT * FROM <entity>`

*args: The remaining positional arguments that are used to bind numeric

references in the query.

**kwds: The dictionary-based arguments for named parameters.

Returns:

A `gql.GqlQuery` object that queries over `BlobInfo`'s datastore kind.

.. _GQL source:

https://cloud.google.com/appengine/docs/python/refdocs/google.appengine.ext.gql

"""

return _GqlQuery('SELECT * FROM %s %s'

% (cls.kind(), query_string),

cls,

*args,

**kwds)

@classmethod

def kind(self):

"""Gets the entity kind for the `BlobInfo`.

Note:

This method is required for compatibility with the current `db.py` query

mechanism but will be removed in the future. DO NOT USE.

Returns:

The entity kind for `BlobInfo`.

"""

return BLOB_INFO_KIND

@classmethod

def __normalize_and_convert_keys(cls, keys):

"""Normalizes and converts all keys to `BlobKey` type.

This method is based on `datastore.NormalizeAndTypeCheck()`.

Args:

keys: A single key or a list or tuple of keys. Keys can be a string or

a `BlobKey`.

Returns:

A single key or a list with all of the strings replaced by `BlobKey`

instances.

"""

if isinstance(keys, (list, tuple)):

multiple = True

keys = list(keys)

else:

multiple = False

keys = [keys]

for index, key in enumerate(keys):

if not isinstance(key, (six.string_types, BlobKey)):

raise datastore_errors.BadArgumentError(

'Expected string or BlobKey; received %s (a %s)' % (

key,

datastore.typename(key)))

keys[index] = datastore.Key.from_path(cls.kind(), str(key), namespace='')

if multiple:

return keys

else:

return keys[0]

def get(blob_key):

"""Gets a `BlobInfo` record from blobstore.

Does the same as `BlobInfo.get`.

Args:

blob_key: The `BlobKey` of the record you want to retrieve.

Returns:

A `BlobInfo` instance that is associated with the provided key or a list

of `BlobInfo` instances if a list of keys was provided. Keys that are not

found in Blobstore return `None`.

"""

return BlobInfo.get(blob_key)

def _get_upload_content(field_storage):

"""Returns an `email.Message` that contains the values of the file transfer.

This method decodes the content of the field storage and creates a new

`email.Message`.

Args:

field_storage: `cgi.FieldStorage` that represents an uploaded blob.

Returns:

An `email.message.Message` that lists the upload information.

"""

message = email.message.Message()

message.add_header(

'content-transfer-encoding',

field_storage.headers.get('Content-Transfer-Encoding', ''))

message.set_payload(field_storage.file.read())

payload = message.get_payload(decode=True)

return email.message_from_string(payload.decode('utf-8'))

def _parse_upload_info(field_storage, error_class):

"""Parses the upload information from the file upload `field_storage`.

Args:

field_storage: `cgi.FieldStorage` that represents the uploaded blob.

error_class: The error to raise if the information cannot be parsed.

Returns:

A dictionary that contains the parsed values. This method will return `None`

if `field_storage` was not defined.

Raises:

`error_class` when provided with a `field_storage` that does not contain

enough information.

"""

if field_storage is None:

return None

field_name = field_storage.name

def get_value(dict, name):

"""Gets the name of the field."""

value = dict.get(name, None)

if value is None:

raise error_class(

'Field %s has no %s.' % (field_name, name))

return value

filename = get_value(field_storage.disposition_options, 'filename')

blob_key = field_storage.type_options.get('blob-key', None)

upload_content = _get_upload_content(field_storage)

field_storage.file.seek(0)

content_type = get_value(upload_content, 'content-type')

size = get_value(upload_content, 'content-length')

creation_string = get_value(upload_content, UPLOAD_INFO_CREATION_HEADER)

md5_hash_encoded = get_value(upload_content, 'content-md5')

md5_hash = base64.urlsafe_b64decode(md5_hash_encoded)

gs_object_name = upload_content.get(CLOUD_STORAGE_OBJECT_HEADER, None)

try:

size = int(size)

except (TypeError, ValueError):

raise error_class(

'%s is not a valid value for %s size.' % (size, field_name))

try:

creation = blobstore._parse_creation(creation_string, field_name)

except blobstore._CreationFormatError as err:

raise error_class(str(err))

return {'blob_key': blob_key,

'content_type': content_type,

'creation': creation,

'filename': filename,

'size': size,

'md5_hash': md5_hash,

'gs_object_name': gs_object_name,

}

def parse_blob_info(field_storage):

"""Parses a `BlobInfo` record from file upload `field_storage`.

Args:

field_storage: `cgi.FieldStorage` that represents an uploaded blob.

Returns:

A `BlobInfo` record as parsed from the `field_storage` instance. This

method will return `None` if `field_storage` was not defined.

Raises:

BlobInfoParseError: If the provided `field_storage` does not contain enough

information to construct a `BlobInfo` object.

"""

info = _parse_upload_info(field_storage, BlobInfoParseError)

if info is None:

return None

key = info.pop('blob_key', None)

if not key:

raise BlobInfoParseError('Field %s has no %s.' % (field_storage.name,

'blob_key'))

return BlobInfo(BlobKey(key), info)

class FileInfo(object):

"""Details information about uploaded files.

This class contains information about blobs stored by an application.

This class is similar to `BlobInfo`; however, this method does not make use of

a key, and the information is not persisted in the datastore.

Properties:

- content_type: The content type of the uploaded file.

- creation: The creation date of the uploaded file, or when it was

uploaded.

- filename: The file name that the user selected from their machine.

- size: The size of the uncompressed file.

- md5_hash: The MD5 hash value of the uploaded file.

- gs_object_name: The name of the file that was written to Google Cloud

Storage, or `None` if the file was not uploaded to Google Cloud Storage.

All properties are read-only. Attempting to assign a value to a property

will raise an `AttributeError`.

"""

def __init__(self, filename=None, content_type=None, creation=None,

size=None, md5_hash=None, gs_object_name=None):

self.__filename = filename

self.__content_type = content_type

self.__creation = creation

self.__size = size

self.__md5_hash = md5_hash

self.__gs_object_name = gs_object_name

@property

def filename(self):

"""Returns the file name that the user selected.

Returns:

The file name that the user selected.

"""

return self.__filename

@property

def content_type(self):

"""Returns the content type of the uploaded file.

Returns:

The content type of the file.

"""

return self.__content_type

@property

def creation(self):

"""Returns the creation date or upload time of the file.

Returns:

The creation date or upload time of the file.

"""

return self.__creation

@property

def size(self):

"""Returns the size of the uncompressed file.

Returns:

The size of the uncompressed file.

"""

return self.__size

@property

def md5_hash(self):

"""Returns the MD5 hash of the uploaded file.

Returns:

The hash value for the uploaded file.

"""

return self.__md5_hash

@property

def gs_object_name(self):

"""Returns the name of the file that was written to Cloud Storage.

Returns:

The name of the file that was written to Cloud Storage.

"""

return self.__gs_object_name

def parse_file_info(field_storage):

"""Parses a `FileInfo` record from file upload `field_storage`.

Args:

field_storage: `cgi.FieldStorage` that represents the uploaded file.

Returns:

`FileInfo` record as parsed from the `field_storage` instance. This method

will return `None` if `field_storage` was not specified.

Raises:

FileInfoParseError: If `field_storage` does not contain enough information

to construct a `FileInfo` object.

"""

info = _parse_upload_info(field_storage, FileInfoParseError)

if info is None:

return None

info.pop('blob_key', None)

return FileInfo(**info)

class BlobReferenceProperty(db.Property):

"""Property compatible with `db.Model` classes.

Add references to blobs to domain models using BlobReferenceProperty::

class Picture(db.Model):

title = db.StringProperty()

image = blobstore.BlobReferenceProperty()

thumbnail = blobstore.BlobReferenceProperty()

To find the size of a picture using this model::

picture = Picture.get(picture_key)

print picture.image.size

`BlobInfo` objects are lazily loaded, so iterating over models for `BlobKeys`

is efficient. The following sample code does not need to hit Datastore for

each image key::

list_of_untitled_blobs = []

for picture in Picture.gql("WHERE title=''"):

list_of_untitled_blobs.append(picture.image.key())

"""

data_type = BlobInfo

def get_value_for_datastore(self, model_instance):

"""Returns a model property translated to a datastore value.

Args:

model_instance: The model property that you want to translate.

Returns:

The model property that was translated from datastore.

"""

blob_info = super(BlobReferenceProperty,

self).get_value_for_datastore(model_instance)

if blob_info is None:

return None

return blob_info.key()

def make_value_from_datastore(self, value):

"""Returns a datastore value to `BlobInfo`.

Args:

value: The datastore value that you want to translate.

Returns:

A `BlobInfo` that was translated from datastore.

"""

if value is None:

return None

return BlobInfo(value)

def validate(self, value):

"""Validates that an assigned value is `BlobInfo`.

This method automatically converts from strings and `BlobKey` instances.

Args:

value: The value that you want to validate.

Returns:

Information about whether an assigned value is `BlobInfo`.

"""

if isinstance(value, (six.string_types)):

value = BlobInfo(BlobKey(value))

elif isinstance(value, BlobKey):

value = BlobInfo(value)

return super(BlobReferenceProperty, self).validate(value)

def fetch_data(blob, start_index, end_index, rpc=None):

"""Fetches data for a blob.

Fetches a fragment of a blob up to `MAX_BLOB_FETCH_SIZE` in length. Attempting

to fetch a fragment that extends beyond the boundaries of the blob will return

the amount of data from `start_index` until the end of the blob, which will be

a smaller size than requested. Requesting a fragment that is entirely outside

the boundaries of the blob will return an empty string. Attempting to fetch a

negative index will raise an exception.

Args:

blob: A `BlobInfo`, `BlobKey`, string, or Unicode representation of

the `BlobKey` of the blob from which you want to fetch data.

start_index: The start index of blob data to fetch. This value must not be

negative.

end_index: The end index (inclusive) of the blob data to fetch. This value

must be greater than or equal to `start_index`.

rpc: Optional UserRPC object.

Returns:

A string that contains partial data of a blob. If the indexes are legal but

outside of the boundaries of the blob, an empty string is returned.

Raises:

TypeError: If `start_index` or `end_index` are not indexes, or if `blob` is

not a string, `BlobKey` or `BlobInfo`.

DataIndexOutOfRangeError: If `start_index` is set to a value that is less

than 0 or `end_index` is less than `start_index`.

BlobFetchSizeTooLargeError: If the requested blob fragment is larger than

`MAX_BLOB_FETCH_SIZE`.

BlobNotFoundError: If the blob does not exist.

"""

rpc = fetch_data_async(blob, start_index, end_index, rpc=rpc)

return rpc.get_result()

def fetch_data_async(blob, start_index, end_index, rpc=None):

"""Asynchronously fetches data for a blob.

Fetches a fragment of a blob up to `MAX_BLOB_FETCH_SIZE` in length. Attempting

to fetch a fragment that extends beyond the boundaries of the blob will return

the amount of data from `start_index` until the end of the blob, which will be

a smaller size than requested. Requesting a fragment that is entirely outside

the boundaries of the blob will return an empty string. Attempting to fetch a

negative index will raise an exception.

Args:

blob: A `BlobInfo`, `BlobKey`, string, or Unicode representation of

the `BlobKey` of the blob from which you want to fetch data.

start_index: The start index of blob data to fetch. This value must not be

negative.

end_index: The end index (inclusive) of the blob data to fetch. This value

must be greater than or equal to `start_index`.

rpc: Optional UserRPC object.

Returns:

A UserRPC whose result will be a string as returned by `fetch_data()`.

Raises:

TypeError: If `start_index` or `end_index` are not indexes, or if `blob` is

not a string, `BlobKey` or `BlobInfo`.

DataIndexOutOfRangeError: If `start_index` is set to a value that is less

than 0 or `end_index` is less than `start_index` when calling

`rpc.get_result()`.

BlobFetchSizeTooLargeError: If the requested blob fragment is larger than

`MAX_BLOB_FETCH_SIZE` when calling `rpc.get_result()`.

BlobNotFoundError: If the blob does not exist when calling

`rpc.get_result()`.

"""

if isinstance(blob, BlobInfo):

blob = blob.key()

return blobstore.fetch_data_async(blob, start_index, end_index, rpc=rpc)

class BlobReader(object):

"""Provides a read-only file-like interface to a blobstore blob."""

SEEK_SET = 0

SEEK_CUR = 1

SEEK_END = 2

def __init__(self, blob, buffer_size=131072, position=0):

"""Constructor.

Args:

blob: The blob key, blob info, or string blob key to read from.

buffer_size: The minimum size to fetch chunks of data from blobstore.

position: The initial position in the file.

Raises:

ValueError: If a `BlobKey`, `BlobInfo`, or string blob key is not

supplied.

"""

if not blob:

raise ValueError('A BlobKey, BlobInfo or string is required.')

if hasattr(blob, 'key'):

self.__blob_key = blob.key()

self.__blob_info = blob

else:

self.__blob_key = blob

self.__blob_info = None

self.__buffer_size = buffer_size

self.__buffer = b''

self.__position = position

self.__buffer_position = 0

self.__eof = False

def __iter__(self):

"""Retrieves a file iterator for this `BlobReader`.

Returns:

The file iterator for the `BlobReader`.

"""

return self

def __getstate__(self):

"""Retrieves the serialized state for this `BlobReader`.

Returns:

The serialized state for the `BlobReader`.

"""

return (self.__blob_key, self.__buffer_size, self.__position)

def __setstate__(self, state):

"""Restores pickled state for this BlobReader."""

self.__init__(*state)

def close(self):

"""Closes the file.

A closed file cannot be read or written to anymore. An operation that

requires that the file to be open will raise a `ValueError` after the file

has been closed. Calling `close()` more than once is allowed.

"""

self.__blob_key = None

def flush(self):

raise IOError("BlobReaders are read-only")

def next(self):

r"""Returns the next line from the file.

Returns:

A string, terminated by `\\n`. The last line cannot end with `\\n`. If the

end of the file is reached, an empty string will be returned.

Raises:

StopIteration: If there are no further lines to read.

"""

line = self.readline()

if not line:

raise StopIteration

return line

def __next__(self):

return self.next()

def __read_from_buffer(self, size):

"""Reads at most `size` bytes from the buffer.

Args:

size: Number of bytes to read, or a negative value to read the entire

buffer.

Returns:

Tuple (data, size):

data: The bytes read from the buffer.

size: The remaining unread byte count. This value is negative when

`size` is negative. Thus when remaining size is not equal to 0, the

calling method might choose to fill the buffer again and keep

reading.

"""

if not self.__blob_key:

raise ValueError("File is closed")

if size < 0:

end_pos = len(self.__buffer)

else:

end_pos = self.__buffer_position + size

data = self.__buffer[self.__buffer_position:end_pos]

data_length = len(data)

size -= data_length

self.__position += data_length

self.__buffer_position += data_length

if self.__buffer_position == len(self.__buffer):

self.__buffer = b''

self.__buffer_position = 0

return data, size

def __fill_buffer(self, size=0):

"""Fills the internal buffer.

Args:

size: Number of bytes to read. Will be clamped to

`[self.__buffer_size, MAX_BLOB_FETCH_SIZE]`.

"""

read_size = min(max(size, self.__buffer_size), MAX_BLOB_FETCH_SIZE)

self.__buffer = fetch_data(self.__blob_key, self.__position,

self.__position + read_size - 1)