Access Control

This Python example can be found at the storage-signed-urls-python repository on GitHub.

# Copyright 2013 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Contains an example of using Google Cloud Storage Signed URLs."""

import base64
import datetime
import md5
import sys
import time

import Crypto.Hash.SHA256 as SHA256
import Crypto.PublicKey.RSA as RSA
import Crypto.Signature.PKCS1_v1_5 as PKCS1_v1_5
import requests

try:
  import conf
except ImportError:
  sys.exit('Configuration module not found. You must create a conf.py file. '
           'See the example in conf.example.py.')

# The Google Cloud Storage API endpoint. You should not need to change this.
GCS_API_ENDPOINT = 'https://storage.googleapis.com'


class CloudStorageURLSigner(object):
  """Contains methods for generating signed URLs for Google Cloud Storage."""

  def __init__(self, key, client_id_email, gcs_api_endpoint, expiration=None,
               session=None):
    """Creates a CloudStorageURLSigner that can be used to access signed URLs.
    Args:
      key: A PyCrypto private key.
      client_id_email: GCS service account email.
      gcs_api_endpoint: Base URL for GCS API.
      expiration: An instance of datetime.datetime containing the time when the
                  signed URL should expire.
      session: A requests.session.Session to use for issuing requests. If not
               supplied, a new session is created.
    """
    self.key = key
    self.client_id_email = client_id_email
    self.gcs_api_endpoint = gcs_api_endpoint

    self.expiration = expiration or (datetime.datetime.now() +
                                     datetime.timedelta(days=1))
    self.expiration = int(time.mktime(self.expiration.timetuple()))

    self.session = session or requests.Session()

  def _Base64Sign(self, plaintext):
    """Signs and returns a base64-encoded SHA256 digest."""
    shahash = SHA256.new(plaintext)
    signer = PKCS1_v1_5.new(self.key)
    signature_bytes = signer.sign(shahash)
    return base64.b64encode(signature_bytes)

  def _MakeSignatureString(self, verb, path, content_md5, content_type):
    """Creates the signature string for signing according to GCS docs."""
    signature_string = ('{verb}\n'
                        '{content_md5}\n'
                        '{content_type}\n'
                        '{expiration}\n'
                        '{resource}')
    return signature_string.format(verb=verb,
                                   content_md5=content_md5,
                                   content_type=content_type,
                                   expiration=self.expiration,
                                   resource=path)

  def _MakeUrl(self, verb, path, content_type='', content_md5=''):
    """Forms and returns the full signed URL to access GCS."""
    base_url = '%s%s' % (self.gcs_api_endpoint, path)
    signature_string = self._MakeSignatureString(verb, path, content_md5,
                                                 content_type)
    signature_signed = self._Base64Sign(signature_string)
    query_params = {'GoogleAccessId': self.client_id_email,
                    'Expires': str(self.expiration),
                    'Signature': signature_signed}
    return base_url, query_params

  def Get(self, path):
    """Performs a GET request.
    Args:
      path: The relative API path to access, e.g. '/bucket/object'.
    Returns:
      An instance of requests.Response containing the HTTP response.
    """
    base_url, query_params = self._MakeUrl('GET', path)
    return self.session.get(base_url, params=query_params)

  def Put(self, path, content_type, data):
    """Performs a PUT request.
    Args:
      path: The relative API path to access, e.g. '/bucket/object'.
      content_type: The content type to assign to the upload.
      data: The file data to upload to the new file.
    Returns:
      An instance of requests.Response containing the HTTP response.
    """
    md5_digest = base64.b64encode(md5.new(data).digest())
    base_url, query_params = self._MakeUrl('PUT', path, content_type,
                                           md5_digest)
    headers = {}
    headers['Content-Type'] = content_type
    headers['Content-Length'] = str(len(data))
    headers['Content-MD5'] = md5_digest
    return self.session.put(base_url, params=query_params, headers=headers,
                            data=data)

  def Delete(self, path):
    """Performs a DELETE request.
    Args:
      path: The relative API path to access, e.g. '/bucket/object'.
    Returns:
      An instance of requests.Response containing the HTTP response.
    """
    base_url, query_params = self._MakeUrl('DELETE', path)
    return self.session.delete(base_url, params=query_params)


def ProcessResponse(r, expected_status=200):
  """Prints request and response information and checks for desired return code.
  Args:
    r: A requests.Response object.
    expected_status: The expected HTTP status code.
  Raises:
    SystemExit if the response code doesn't match expected_status.
  """
  print '--- Request ---'
  print r.request.full_url
  for header, value in r.request.headers.iteritems():
    print '%s: %s' % (header, value)
  print '---------------'
  print '--- Response (Status %s) ---' % r.status_code
  print r.content
  print '-----------------------------'
  print
  if r.status_code != expected_status:
    sys.exit('Exiting due to receiving %d status code when expecting %d.'
             % (r.status_code, expected_status))


def main():
  try:
    keytext = open(conf.PRIVATE_KEY_PATH, 'rb').read()
  except IOError as e:
    sys.exit('Error while reading private key: %s' % e)

  private_key = RSA.importKey(keytext)
  signer = CloudStorageURLSigner(private_key, conf.SERVICE_ACCOUNT_EMAIL,
                                 GCS_API_ENDPOINT)

  file_path = '/%s/%s' % (conf.BUCKET_NAME, conf.OBJECT_NAME)

  print 'Creating file...'
  print '================'
  r = signer.Put(file_path, 'text/plain', 'blah blah')
  ProcessResponse(r)
  print 'Retrieving file...'
  print '=================='
  r = signer.Get(file_path)
  ProcessResponse(r)
  print 'Deleting file...'
  print '================'
  r = signer.Delete(file_path)
  ProcessResponse(r, expected_status=204)
  print 'Done.'

if __name__ == '__main__':
  main()

/*
 * Copyright (c) 2014 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

// Package main contains a program that demonstrates various Google Cloud Storage Signed URL operations.
// More information can be found at
// https://cloud.google.com/storage/docs/access-control#Signed-URLs
// http://godoc.org/google.golang.org/cloud/storage#SignedURL
// Replace googleAccessID,serviceAccountPEMFilename,bucket constants
// Note: default bucketname for App Engine projects is formatted as: <app_id>.appspot.com
// Convert your PKCS12 private key file to PEM:
//
//    openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes
//
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"google.golang.org/cloud/storage"
)

const (
	googleAccessID            = "<serviceAccountEmail>@developer.gserviceaccount.com"
	serviceAccountPEMFilename = "YOUR_SERVICE_ACCOUNT_KEY.pem"
	bucket                    = "YOURBUCKET"
	object                    = "somerandomfile.txt"
)

var (
	expiration = time.Now().Add(time.Second * 60) //expire in 60 seconds
)

func main() {
	data, err := ioutil.ReadFile(serviceAccountPEMFilename)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("Putting the object")
	opts := &storage.SignedURLOptions{
		ClientID:   googleAccessID,
		PrivateKey: data,
		Method:     "PUT",
		Expires:    expiration,
	}

	putURL, err := storage.SignedURL(bucket, object, opts)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("PUT URL : %v\n", putURL)

	client := &http.Client{}
	var payload = []byte("Lorem Ipsum")
	req, err := http.NewRequest("PUT", putURL, bytes.NewBuffer(payload))
	res, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	res.Body.Close()
	fmt.Printf("Response Code: %s\n", res.Status)

	fmt.Println("Getting the object")
	opts = &storage.SignedURLOptions{
		ClientID:   googleAccessID,
		PrivateKey: data,
		Method:     "GET",
		Expires:    expiration,
	}

	getURL, err := storage.SignedURL(bucket, object, opts)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("GET URL : %v\n", getURL)

	res, err = http.Get(getURL)
	if err != nil {
		log.Fatal(err)
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	res.Body.Close()
	fmt.Printf("Response Code: %s\n", res.Status)
	fmt.Printf("%s\n", body)

	fmt.Println("Deleting the object")
	opts = &storage.SignedURLOptions{
		ClientID:   googleAccessID,
		PrivateKey: data,
		Method:     "DELETE",
		Expires:    expiration,
	}

	deleteURL, err := storage.SignedURL(bucket, object, opts)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("DELETE URL : %v\n", deleteURL)
	req, err = http.NewRequest("DELETE", deleteURL, nil)
	res, err = client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	res.Body.Close()
	fmt.Printf("Response Code: %s\n", res.Status)

}

/*
 * Copyright (c) 2015 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

/**
 * Sample code to issue several basic Google Cloud Store (GCS) operations
 * using the Google Client Libraries.
 * For more information, see documentation for Compute Storage .NET client
 * https://developers.google.com/api-client-library/dotnet/apis/storage/v1
 *
 * Usage:
 * Add projects references using NuGet to
 * "Google.Apis.Storage.v1"   (other dependencies will get added automatically)
 * http://www.nuget.org/packages/Google.Apis.Storage.v1/
 * Generate and download service account .p12 from place it to c:\
 * specify the bucketName, projectId, serviceAccountEmail and certificateFile below.
 * authorize serviceAccountEmail for your target bucket.
 */

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

using Google.Apis;
using Google.Apis.Auth.OAuth2;
using Google.Apis.Download;
using Google.Apis.Services;
using Google.Apis.Storage.v1;
using Google.Apis.Storage.v1.Data;
using Google.Apis.Util.Store;

namespace GCSSearch
{
    internal class Program
    {
        [STAThread]
        static void Main(string[] args)
        {
            try
            {
                new Program().Run().Wait();
            }
            catch (AggregateException ex)
            {
                foreach (var err in ex.InnerExceptions)
                {
                    Console.WriteLine("ERROR: " + err.Message);
                }
            }
            Console.ReadKey();
        }

        #region Consts

        private const int KB = 0x400;
        private const int DownloadChunkSize = 256 * KB;

        #endregion

        private async Task Run()
        {
            string projectId = "YOUR_PROJECT_ID";
            string bucketName = "YOUR_BUCKET_NAME";

            //credentials for certificate-based service accounts
            string certificateFile = "c:\\YOUR_CERTIFICATE.p12";
            string serviceAccountEmail = "YOUR_SERVICE_ACCOUNT_EMAIL@developer.gserviceaccount.com";

            var certificate = new X509Certificate2(certificateFile, "notasecret", X509KeyStorageFlags.Exportable);
            ServiceAccountCredential credential = new ServiceAccountCredential(
               new ServiceAccountCredential.Initializer(serviceAccountEmail)
               {
                   Scopes = new[] { StorageService.Scope.DevstorageReadWrite }
               }.FromCertificate(certificate));


            //credentials when running in google compute engine
            //ComputeCredential credential = new ComputeCredential(new ComputeCredential.Initializer());

            //credentials for user interactive webflow
            /*
            UserCredential credential;
            string clientId = "YOUR_CLIENT_ID.apps.googleusercontent.com";
            string clientSecret = "YOUR_CLIENT_SECRET";
            credential = await GoogleWebAuthorizationBroker.AuthorizeAsync(new ClientSecrets{ClientId= clientId,ClientSecret=clientSecret},
                new[] { StorageService.Scope.DevstorageFullControl }, Environment.UserName, CancellationToken.None);
            //Console.WriteLine("Credential file saved at: " + Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
             */

            var service = new StorageService(new BaseClientService.Initializer()
            {
                HttpClientInitializer = credential,
                ApplicationName = "GCS Sample",
            });

            Console.WriteLine("List of buckets in current project");
            Buckets buckets = await service.Buckets.List(projectId).ExecuteAsync();

            foreach (var bkt in buckets.Items)
            {
                Console.WriteLine(bkt.Name);
            }

            Console.WriteLine("Total number of items in bucket: " + buckets.Items.Count);
            Console.WriteLine("=============================");

            // using  Google.Apis.Storage.v1.Data.Object to disambiguate from System.Object
            Google.Apis.Storage.v1.Data.Object fileobj = new Google.Apis.Storage.v1.Data.Object() { Name = "somefile.txt" };

            Console.WriteLine("Creating " + fileobj.Name + " in bucket " + bucketName);
            byte[] msgtxt = Encoding.UTF8.GetBytes("Lorem Ipsum");

            using (var streamOut = new MemoryStream(msgtxt))
            {
                await service.Objects.Insert(fileobj, bucketName, streamOut, "text/plain").UploadAsync();
            }

            Console.WriteLine("Object created: " + fileobj.Name);

            Console.WriteLine("=============================");

            Console.WriteLine("Reading object " + fileobj.Name + " in bucket: " + bucketName);
            var req = service.Objects.Get(bucketName, fileobj.Name);
            Google.Apis.Storage.v1.Data.Object readobj = await req.ExecuteAsync();

            Console.WriteLine("Object MediaLink: " + readobj.MediaLink);

            // download using Google.Apis.Download and display the progress
            string pathUser = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
            string downloadToDirectory;
            // create the directory and path if the GCS file needs to get saved under a subdirectory
            if (Path.GetDirectoryName(readobj.Name) == String.Empty)
                downloadToDirectory = Path.Combine(pathUser, "Downloads");
            else
                downloadToDirectory = Path.Combine(pathUser, "Downloads", Path.GetDirectoryName(readobj.Name));
            Directory.CreateDirectory(downloadToDirectory);
            var fileName = Path.Combine(downloadToDirectory,Path.GetFileName(readobj.Name));
            Console.WriteLine("Starting download to " + fileName);
            var downloader = new MediaDownloader(service)
            {
                ChunkSize = DownloadChunkSize
            };
            // add a delegate for the progress changed event for writing to console on changes
            downloader.ProgressChanged += Download_ProgressChanged;

            using (var fileStream = new System.IO.FileStream(fileName,
                System.IO.FileMode.Create, System.IO.FileAccess.Write))
            {
                var progress = await downloader.DownloadAsync(readobj.MediaLink, fileStream);
                if (progress.Status == DownloadStatus.Completed)
                {
                    Console.WriteLine(readobj.Name + " was downloaded successfully");
                }
                else
                {
                    Console.WriteLine("Download {0} was interpreted in the middle. Only {1} were downloaded. ",
                        readobj.Name, progress.BytesDownloaded);
                }
            }

            /*
            // or download as a stream
            Stream getStream = await service.HttpClient.GetStreamAsync(readobj.MediaLink);
            Console.WriteLine("Object Content: ");
            using (var reader = new StreamReader(getStream))
            {
                Console.WriteLine(reader.ReadToEnd());
            }
            */
            Console.WriteLine("=============================");
        }

        #region Progress and Response changes

        static void Download_ProgressChanged(IDownloadProgress progress)
        {
            Console.WriteLine(progress.Status + " " + progress.BytesDownloaded + " bytes");
        }

        #endregion
    }
}