Skip to content

Commit 980c114

Browse files
authored
Clean up legacy references to apitools in GCS I/O (#39433)
1 parent b8d77b8 commit 980c114

3 files changed

Lines changed: 5 additions & 24 deletions

File tree

sdks/python/apache_beam/io/filesystemio.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,9 +284,8 @@ def tell(self):
284284
return self.position
285285

286286
def seek(self, offset, whence=os.SEEK_SET):
287-
# The apitools library used by the gcsio.Uploader class insists on seeking
288-
# to the end of a stream to do a check before completing an upload, so we
289-
# must have this no-op method here in that case.
287+
# Certain upload stream implementations seek to the end of a stream to check
288+
# length before completing an upload, so we support a no-op seek(0, SEEK_END).
290289
if whence == os.SEEK_END and offset == 0:
291290
return
292291
elif whence == os.SEEK_SET:

sdks/python/apache_beam/io/gcp/__init__.py

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,3 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616
#
17-
18-
# Important: the MIME library in the Python 3.x standard library used by
19-
# apitools causes uploads containing '\r\n' to be corrupted, unless we
20-
# patch the BytesGenerator class to write contents verbatim.
21-
try:
22-
# pylint: disable=wrong-import-order, wrong-import-position
23-
# pylint: disable=ungrouped-imports
24-
import email.generator as email_generator
25-
26-
from apitools.base.py import transfer
27-
28-
class _WrapperNamespace(object):
29-
class BytesGenerator(email_generator.BytesGenerator):
30-
def _write_lines(self, lines):
31-
self.write(lines)
32-
33-
transfer.email_generator = _WrapperNamespace
34-
except ImportError:
35-
# We may not have the GCP dependencies installed, so we pass in this case.
36-
pass

sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,11 @@
2929
from apache_beam.io.filesystem import FileMetadata
3030
from apache_beam.options.pipeline_options import PipelineOptions
3131

32-
# Protect against environments where apitools library is not available.
32+
# Protect against environments where GCP storage library is not available.
3333
# pylint: disable=wrong-import-order, wrong-import-position
3434
try:
35+
from google.cloud import storage # pylint: disable=unused-import
36+
3537
from apache_beam.io.gcp import gcsfilesystem
3638
except ImportError:
3739
gcsfilesystem = None # type: ignore

0 commit comments

Comments
 (0)