parent
11a731ba82
commit
313ab48b45
@ -1,202 +0,0 @@ |
||||
|
||||
Apache License |
||||
Version 2.0, January 2004 |
||||
http://www.apache.org/licenses/ |
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||
|
||||
1. Definitions. |
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, |
||||
and distribution as defined by Sections 1 through 9 of this document. |
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by |
||||
the copyright owner that is granting the License. |
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all |
||||
other entities that control, are controlled by, or are under common |
||||
control with that entity. For the purposes of this definition, |
||||
"control" means (i) the power, direct or indirect, to cause the |
||||
direction or management of such entity, whether by contract or |
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||
outstanding shares, or (iii) beneficial ownership of such entity. |
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity |
||||
exercising permissions granted by this License. |
||||
|
||||
"Source" form shall mean the preferred form for making modifications, |
||||
including but not limited to software source code, documentation |
||||
source, and configuration files. |
||||
|
||||
"Object" form shall mean any form resulting from mechanical |
||||
transformation or translation of a Source form, including but |
||||
not limited to compiled object code, generated documentation, |
||||
and conversions to other media types. |
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or |
||||
Object form, made available under the License, as indicated by a |
||||
copyright notice that is included in or attached to the work |
||||
(an example is provided in the Appendix below). |
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object |
||||
form, that is based on (or derived from) the Work and for which the |
||||
editorial revisions, annotations, elaborations, or other modifications |
||||
represent, as a whole, an original work of authorship. For the purposes |
||||
of this License, Derivative Works shall not include works that remain |
||||
separable from, or merely link (or bind by name) to the interfaces of, |
||||
the Work and Derivative Works thereof. |
||||
|
||||
"Contribution" shall mean any work of authorship, including |
||||
the original version of the Work and any modifications or additions |
||||
to that Work or Derivative Works thereof, that is intentionally |
||||
submitted to Licensor for inclusion in the Work by the copyright owner |
||||
or by an individual or Legal Entity authorized to submit on behalf of |
||||
the copyright owner. For the purposes of this definition, "submitted" |
||||
means any form of electronic, verbal, or written communication sent |
||||
to the Licensor or its representatives, including but not limited to |
||||
communication on electronic mailing lists, source code control systems, |
||||
and issue tracking systems that are managed by, or on behalf of, the |
||||
Licensor for the purpose of discussing and improving the Work, but |
||||
excluding communication that is conspicuously marked or otherwise |
||||
designated in writing by the copyright owner as "Not a Contribution." |
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||
on behalf of whom a Contribution has been received by Licensor and |
||||
subsequently incorporated within the Work. |
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
copyright license to reproduce, prepare Derivative Works of, |
||||
publicly display, publicly perform, sublicense, and distribute the |
||||
Work and such Derivative Works in Source or Object form. |
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
(except as stated in this section) patent license to make, have made, |
||||
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||
where such license applies only to those patent claims licensable |
||||
by such Contributor that are necessarily infringed by their |
||||
Contribution(s) alone or by combination of their Contribution(s) |
||||
with the Work to which such Contribution(s) was submitted. If You |
||||
institute patent litigation against any entity (including a |
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||
or a Contribution incorporated within the Work constitutes direct |
||||
or contributory patent infringement, then any patent licenses |
||||
granted to You under this License for that Work shall terminate |
||||
as of the date such litigation is filed. |
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the |
||||
Work or Derivative Works thereof in any medium, with or without |
||||
modifications, and in Source or Object form, provided that You |
||||
meet the following conditions: |
||||
|
||||
(a) You must give any other recipients of the Work or |
||||
Derivative Works a copy of this License; and |
||||
|
||||
(b) You must cause any modified files to carry prominent notices |
||||
stating that You changed the files; and |
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works |
||||
that You distribute, all copyright, patent, trademark, and |
||||
attribution notices from the Source form of the Work, |
||||
excluding those notices that do not pertain to any part of |
||||
the Derivative Works; and |
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its |
||||
distribution, then any Derivative Works that You distribute must |
||||
include a readable copy of the attribution notices contained |
||||
within such NOTICE file, excluding those notices that do not |
||||
pertain to any part of the Derivative Works, in at least one |
||||
of the following places: within a NOTICE text file distributed |
||||
as part of the Derivative Works; within the Source form or |
||||
documentation, if provided along with the Derivative Works; or, |
||||
within a display generated by the Derivative Works, if and |
||||
wherever such third-party notices normally appear. The contents |
||||
of the NOTICE file are for informational purposes only and |
||||
do not modify the License. You may add Your own attribution |
||||
notices within Derivative Works that You distribute, alongside |
||||
or as an addendum to the NOTICE text from the Work, provided |
||||
that such additional attribution notices cannot be construed |
||||
as modifying the License. |
||||
|
||||
You may add Your own copyright statement to Your modifications and |
||||
may provide additional or different license terms and conditions |
||||
for use, reproduction, or distribution of Your modifications, or |
||||
for any such Derivative Works as a whole, provided Your use, |
||||
reproduction, and distribution of the Work otherwise complies with |
||||
the conditions stated in this License. |
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||
any Contribution intentionally submitted for inclusion in the Work |
||||
by You to the Licensor shall be under the terms and conditions of |
||||
this License, without any additional terms or conditions. |
||||
Notwithstanding the above, nothing herein shall supersede or modify |
||||
the terms of any separate license agreement you may have executed |
||||
with Licensor regarding such Contributions. |
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade |
||||
names, trademarks, service marks, or product names of the Licensor, |
||||
except as required for reasonable and customary use in describing the |
||||
origin of the Work and reproducing the content of the NOTICE file. |
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or |
||||
agreed to in writing, Licensor provides the Work (and each |
||||
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||
implied, including, without limitation, any warranties or conditions |
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||
appropriateness of using or redistributing the Work and assume any |
||||
risks associated with Your exercise of permissions under this License. |
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, |
||||
whether in tort (including negligence), contract, or otherwise, |
||||
unless required by applicable law (such as deliberate and grossly |
||||
negligent acts) or agreed to in writing, shall any Contributor be |
||||
liable to You for damages, including any direct, indirect, special, |
||||
incidental, or consequential damages of any character arising as a |
||||
result of this License or out of the use or inability to use the |
||||
Work (including but not limited to damages for loss of goodwill, |
||||
work stoppage, computer failure or malfunction, or any and all |
||||
other commercial damages or losses), even if such Contributor |
||||
has been advised of the possibility of such damages. |
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing |
||||
the Work or Derivative Works thereof, You may choose to offer, |
||||
and charge a fee for, acceptance of support, warranty, indemnity, |
||||
or other liability obligations and/or rights consistent with this |
||||
License. However, in accepting such obligations, You may act only |
||||
on Your own behalf and on Your sole responsibility, not on behalf |
||||
of any other Contributor, and only if You agree to indemnify, |
||||
defend, and hold each Contributor harmless for any liability |
||||
incurred by, or claims asserted against, such Contributor by reason |
||||
of your accepting any such warranty or additional liability. |
||||
|
||||
END OF TERMS AND CONDITIONS |
||||
|
||||
APPENDIX: How to apply the Apache License to your work. |
||||
|
||||
To apply the Apache License to your work, attach the following |
||||
boilerplate notice, with the fields enclosed by brackets "[]" |
||||
replaced with your own identifying information. (Don't include |
||||
the brackets!) The text should be enclosed in the appropriate |
||||
comment syntax for the file format. We also recommend that a |
||||
file or class name and description of purpose be included on the |
||||
same "printed page" as the copyright notice for easier |
||||
identification within third-party archives. |
||||
|
||||
Copyright 2014 Google Inc. |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
@ -1,438 +0,0 @@ |
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package metadata provides access to Google Compute Engine (GCE)
|
||||
// metadata and API service accounts.
|
||||
//
|
||||
// This package is a wrapper around the GCE metadata service,
|
||||
// as documented at https://developers.google.com/compute/docs/metadata.
|
||||
package metadata |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net" |
||||
"net/http" |
||||
"net/url" |
||||
"os" |
||||
"runtime" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"golang.org/x/net/context" |
||||
"golang.org/x/net/context/ctxhttp" |
||||
|
||||
"cloud.google.com/go/internal" |
||||
) |
||||
|
||||
const ( |
||||
// metadataIP is the documented metadata server IP address.
|
||||
metadataIP = "169.254.169.254" |
||||
|
||||
// metadataHostEnv is the environment variable specifying the
|
||||
// GCE metadata hostname. If empty, the default value of
|
||||
// metadataIP ("169.254.169.254") is used instead.
|
||||
// This is variable name is not defined by any spec, as far as
|
||||
// I know; it was made up for the Go package.
|
||||
metadataHostEnv = "GCE_METADATA_HOST" |
||||
) |
||||
|
||||
type cachedValue struct { |
||||
k string |
||||
trim bool |
||||
mu sync.Mutex |
||||
v string |
||||
} |
||||
|
||||
var ( |
||||
projID = &cachedValue{k: "project/project-id", trim: true} |
||||
projNum = &cachedValue{k: "project/numeric-project-id", trim: true} |
||||
instID = &cachedValue{k: "instance/id", trim: true} |
||||
) |
||||
|
||||
var ( |
||||
metaClient = &http.Client{ |
||||
Transport: &internal.Transport{ |
||||
Base: &http.Transport{ |
||||
Dial: (&net.Dialer{ |
||||
Timeout: 2 * time.Second, |
||||
KeepAlive: 30 * time.Second, |
||||
}).Dial, |
||||
ResponseHeaderTimeout: 2 * time.Second, |
||||
}, |
||||
}, |
||||
} |
||||
subscribeClient = &http.Client{ |
||||
Transport: &internal.Transport{ |
||||
Base: &http.Transport{ |
||||
Dial: (&net.Dialer{ |
||||
Timeout: 2 * time.Second, |
||||
KeepAlive: 30 * time.Second, |
||||
}).Dial, |
||||
}, |
||||
}, |
||||
} |
||||
) |
||||
|
||||
// NotDefinedError is returned when requested metadata is not defined.
|
||||
//
|
||||
// The underlying string is the suffix after "/computeMetadata/v1/".
|
||||
//
|
||||
// This error is not returned if the value is defined to be the empty
|
||||
// string.
|
||||
type NotDefinedError string |
||||
|
||||
func (suffix NotDefinedError) Error() string { |
||||
return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) |
||||
} |
||||
|
||||
// Get returns a value from the metadata service.
|
||||
// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
|
||||
//
|
||||
// If the GCE_METADATA_HOST environment variable is not defined, a default of
|
||||
// 169.254.169.254 will be used instead.
|
||||
//
|
||||
// If the requested metadata is not defined, the returned error will
|
||||
// be of type NotDefinedError.
|
||||
func Get(suffix string) (string, error) { |
||||
val, _, err := getETag(metaClient, suffix) |
||||
return val, err |
||||
} |
||||
|
||||
// getETag returns a value from the metadata service as well as the associated
|
||||
// ETag using the provided client. This func is otherwise equivalent to Get.
|
||||
func getETag(client *http.Client, suffix string) (value, etag string, err error) { |
||||
// Using a fixed IP makes it very difficult to spoof the metadata service in
|
||||
// a container, which is an important use-case for local testing of cloud
|
||||
// deployments. To enable spoofing of the metadata service, the environment
|
||||
// variable GCE_METADATA_HOST is first inspected to decide where metadata
|
||||
// requests shall go.
|
||||
host := os.Getenv(metadataHostEnv) |
||||
if host == "" { |
||||
// Using 169.254.169.254 instead of "metadata" here because Go
|
||||
// binaries built with the "netgo" tag and without cgo won't
|
||||
// know the search suffix for "metadata" is
|
||||
// ".google.internal", and this IP address is documented as
|
||||
// being stable anyway.
|
||||
host = metadataIP |
||||
} |
||||
url := "http://" + host + "/computeMetadata/v1/" + suffix |
||||
req, _ := http.NewRequest("GET", url, nil) |
||||
req.Header.Set("Metadata-Flavor", "Google") |
||||
res, err := client.Do(req) |
||||
if err != nil { |
||||
return "", "", err |
||||
} |
||||
defer res.Body.Close() |
||||
if res.StatusCode == http.StatusNotFound { |
||||
return "", "", NotDefinedError(suffix) |
||||
} |
||||
if res.StatusCode != 200 { |
||||
return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) |
||||
} |
||||
all, err := ioutil.ReadAll(res.Body) |
||||
if err != nil { |
||||
return "", "", err |
||||
} |
||||
return string(all), res.Header.Get("Etag"), nil |
||||
} |
||||
|
||||
func getTrimmed(suffix string) (s string, err error) { |
||||
s, err = Get(suffix) |
||||
s = strings.TrimSpace(s) |
||||
return |
||||
} |
||||
|
||||
func (c *cachedValue) get() (v string, err error) { |
||||
defer c.mu.Unlock() |
||||
c.mu.Lock() |
||||
if c.v != "" { |
||||
return c.v, nil |
||||
} |
||||
if c.trim { |
||||
v, err = getTrimmed(c.k) |
||||
} else { |
||||
v, err = Get(c.k) |
||||
} |
||||
if err == nil { |
||||
c.v = v |
||||
} |
||||
return |
||||
} |
||||
|
||||
var ( |
||||
onGCEOnce sync.Once |
||||
onGCE bool |
||||
) |
||||
|
||||
// OnGCE reports whether this process is running on Google Compute Engine.
|
||||
func OnGCE() bool { |
||||
onGCEOnce.Do(initOnGCE) |
||||
return onGCE |
||||
} |
||||
|
||||
func initOnGCE() { |
||||
onGCE = testOnGCE() |
||||
} |
||||
|
||||
func testOnGCE() bool { |
||||
// The user explicitly said they're on GCE, so trust them.
|
||||
if os.Getenv(metadataHostEnv) != "" { |
||||
return true |
||||
} |
||||
|
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
defer cancel() |
||||
|
||||
resc := make(chan bool, 2) |
||||
|
||||
// Try two strategies in parallel.
|
||||
// See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194
|
||||
go func() { |
||||
res, err := ctxhttp.Get(ctx, metaClient, "http://"+metadataIP) |
||||
if err != nil { |
||||
resc <- false |
||||
return |
||||
} |
||||
defer res.Body.Close() |
||||
resc <- res.Header.Get("Metadata-Flavor") == "Google" |
||||
}() |
||||
|
||||
go func() { |
||||
addrs, err := net.LookupHost("metadata.google.internal") |
||||
if err != nil || len(addrs) == 0 { |
||||
resc <- false |
||||
return |
||||
} |
||||
resc <- strsContains(addrs, metadataIP) |
||||
}() |
||||
|
||||
tryHarder := systemInfoSuggestsGCE() |
||||
if tryHarder { |
||||
res := <-resc |
||||
if res { |
||||
// The first strategy succeeded, so let's use it.
|
||||
return true |
||||
} |
||||
// Wait for either the DNS or metadata server probe to
|
||||
// contradict the other one and say we are running on
|
||||
// GCE. Give it a lot of time to do so, since the system
|
||||
// info already suggests we're running on a GCE BIOS.
|
||||
timer := time.NewTimer(5 * time.Second) |
||||
defer timer.Stop() |
||||
select { |
||||
case res = <-resc: |
||||
return res |
||||
case <-timer.C: |
||||
// Too slow. Who knows what this system is.
|
||||
return false |
||||
} |
||||
} |
||||
|
||||
// There's no hint from the system info that we're running on
|
||||
// GCE, so use the first probe's result as truth, whether it's
|
||||
// true or false. The goal here is to optimize for speed for
|
||||
// users who are NOT running on GCE. We can't assume that
|
||||
// either a DNS lookup or an HTTP request to a blackholed IP
|
||||
// address is fast. Worst case this should return when the
|
||||
// metaClient's Transport.ResponseHeaderTimeout or
|
||||
// Transport.Dial.Timeout fires (in two seconds).
|
||||
return <-resc |
||||
} |
||||
|
||||
// systemInfoSuggestsGCE reports whether the local system (without
|
||||
// doing network requests) suggests that we're running on GCE. If this
|
||||
// returns true, testOnGCE tries a bit harder to reach its metadata
|
||||
// server.
|
||||
func systemInfoSuggestsGCE() bool { |
||||
if runtime.GOOS != "linux" { |
||||
// We don't have any non-Linux clues available, at least yet.
|
||||
return false |
||||
} |
||||
slurp, _ := ioutil.ReadFile("/sys/class/dmi/id/product_name") |
||||
name := strings.TrimSpace(string(slurp)) |
||||
return name == "Google" || name == "Google Compute Engine" |
||||
} |
||||
|
||||
// Subscribe subscribes to a value from the metadata service.
|
||||
// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/".
|
||||
// The suffix may contain query parameters.
|
||||
//
|
||||
// Subscribe calls fn with the latest metadata value indicated by the provided
|
||||
// suffix. If the metadata value is deleted, fn is called with the empty string
|
||||
// and ok false. Subscribe blocks until fn returns a non-nil error or the value
|
||||
// is deleted. Subscribe returns the error value returned from the last call to
|
||||
// fn, which may be nil when ok == false.
|
||||
func Subscribe(suffix string, fn func(v string, ok bool) error) error { |
||||
const failedSubscribeSleep = time.Second * 5 |
||||
|
||||
// First check to see if the metadata value exists at all.
|
||||
val, lastETag, err := getETag(subscribeClient, suffix) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := fn(val, true); err != nil { |
||||
return err |
||||
} |
||||
|
||||
ok := true |
||||
if strings.ContainsRune(suffix, '?') { |
||||
suffix += "&wait_for_change=true&last_etag=" |
||||
} else { |
||||
suffix += "?wait_for_change=true&last_etag=" |
||||
} |
||||
for { |
||||
val, etag, err := getETag(subscribeClient, suffix+url.QueryEscape(lastETag)) |
||||
if err != nil { |
||||
if _, deleted := err.(NotDefinedError); !deleted { |
||||
time.Sleep(failedSubscribeSleep) |
||||
continue // Retry on other errors.
|
||||
} |
||||
ok = false |
||||
} |
||||
lastETag = etag |
||||
|
||||
if err := fn(val, ok); err != nil || !ok { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
// ProjectID returns the current instance's project ID string.
|
||||
func ProjectID() (string, error) { return projID.get() } |
||||
|
||||
// NumericProjectID returns the current instance's numeric project ID.
|
||||
func NumericProjectID() (string, error) { return projNum.get() } |
||||
|
||||
// InternalIP returns the instance's primary internal IP address.
|
||||
func InternalIP() (string, error) { |
||||
return getTrimmed("instance/network-interfaces/0/ip") |
||||
} |
||||
|
||||
// ExternalIP returns the instance's primary external (public) IP address.
|
||||
func ExternalIP() (string, error) { |
||||
return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") |
||||
} |
||||
|
||||
// Hostname returns the instance's hostname. This will be of the form
|
||||
// "<instanceID>.c.<projID>.internal".
|
||||
func Hostname() (string, error) { |
||||
return getTrimmed("instance/hostname") |
||||
} |
||||
|
||||
// InstanceTags returns the list of user-defined instance tags,
|
||||
// assigned when initially creating a GCE instance.
|
||||
func InstanceTags() ([]string, error) { |
||||
var s []string |
||||
j, err := Get("instance/tags") |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { |
||||
return nil, err |
||||
} |
||||
return s, nil |
||||
} |
||||
|
||||
// InstanceID returns the current VM's numeric instance ID.
|
||||
func InstanceID() (string, error) { |
||||
return instID.get() |
||||
} |
||||
|
||||
// InstanceName returns the current VM's instance ID string.
|
||||
func InstanceName() (string, error) { |
||||
host, err := Hostname() |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
return strings.Split(host, ".")[0], nil |
||||
} |
||||
|
||||
// Zone returns the current VM's zone, such as "us-central1-b".
|
||||
func Zone() (string, error) { |
||||
zone, err := getTrimmed("instance/zone") |
||||
// zone is of the form "projects/<projNum>/zones/<zoneName>".
|
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
return zone[strings.LastIndex(zone, "/")+1:], nil |
||||
} |
||||
|
||||
// InstanceAttributes returns the list of user-defined attributes,
|
||||
// assigned when initially creating a GCE VM instance. The value of an
|
||||
// attribute can be obtained with InstanceAttributeValue.
|
||||
func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") } |
||||
|
||||
// ProjectAttributes returns the list of user-defined attributes
|
||||
// applying to the project as a whole, not just this VM. The value of
|
||||
// an attribute can be obtained with ProjectAttributeValue.
|
||||
func ProjectAttributes() ([]string, error) { return lines("project/attributes/") } |
||||
|
||||
func lines(suffix string) ([]string, error) { |
||||
j, err := Get(suffix) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
s := strings.Split(strings.TrimSpace(j), "\n") |
||||
for i := range s { |
||||
s[i] = strings.TrimSpace(s[i]) |
||||
} |
||||
return s, nil |
||||
} |
||||
|
||||
// InstanceAttributeValue returns the value of the provided VM
|
||||
// instance attribute.
|
||||
//
|
||||
// If the requested attribute is not defined, the returned error will
|
||||
// be of type NotDefinedError.
|
||||
//
|
||||
// InstanceAttributeValue may return ("", nil) if the attribute was
|
||||
// defined to be the empty string.
|
||||
func InstanceAttributeValue(attr string) (string, error) { |
||||
return Get("instance/attributes/" + attr) |
||||
} |
||||
|
||||
// ProjectAttributeValue returns the value of the provided
|
||||
// project attribute.
|
||||
//
|
||||
// If the requested attribute is not defined, the returned error will
|
||||
// be of type NotDefinedError.
|
||||
//
|
||||
// ProjectAttributeValue may return ("", nil) if the attribute was
|
||||
// defined to be the empty string.
|
||||
func ProjectAttributeValue(attr string) (string, error) { |
||||
return Get("project/attributes/" + attr) |
||||
} |
||||
|
||||
// Scopes returns the service account scopes for the given account.
|
||||
// The account may be empty or the string "default" to use the instance's
|
||||
// main account.
|
||||
func Scopes(serviceAccount string) ([]string, error) { |
||||
if serviceAccount == "" { |
||||
serviceAccount = "default" |
||||
} |
||||
return lines("instance/service-accounts/" + serviceAccount + "/scopes") |
||||
} |
||||
|
||||
func strsContains(ss []string, s string) bool { |
||||
for _, v := range ss { |
||||
if v == s { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
@ -1,64 +0,0 @@ |
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package internal provides support for the cloud packages.
|
||||
//
|
||||
// Users should not import this package directly.
|
||||
package internal |
||||
|
||||
import ( |
||||
"fmt" |
||||
"net/http" |
||||
) |
||||
|
||||
const userAgent = "gcloud-golang/0.1" |
||||
|
||||
// Transport is an http.RoundTripper that appends Google Cloud client's
|
||||
// user-agent to the original request's user-agent header.
|
||||
type Transport struct { |
||||
// TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does.
|
||||
// Do User-Agent some other way.
|
||||
|
||||
// Base is the actual http.RoundTripper
|
||||
// requests will use. It must not be nil.
|
||||
Base http.RoundTripper |
||||
} |
||||
|
||||
// RoundTrip appends a user-agent to the existing user-agent
|
||||
// header and delegates the request to the base http.RoundTripper.
|
||||
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { |
||||
req = cloneRequest(req) |
||||
ua := req.Header.Get("User-Agent") |
||||
if ua == "" { |
||||
ua = userAgent |
||||
} else { |
||||
ua = fmt.Sprintf("%s %s", ua, userAgent) |
||||
} |
||||
req.Header.Set("User-Agent", ua) |
||||
return t.Base.RoundTrip(req) |
||||
} |
||||
|
||||
// cloneRequest returns a clone of the provided *http.Request.
|
||||
// The clone is a shallow copy of the struct and its Header map.
|
||||
func cloneRequest(r *http.Request) *http.Request { |
||||
// shallow copy of the struct
|
||||
r2 := new(http.Request) |
||||
*r2 = *r |
||||
// deep copy of the Header
|
||||
r2.Header = make(http.Header) |
||||
for k, s := range r.Header { |
||||
r2.Header[k] = s |
||||
} |
||||
return r2 |
||||
} |
||||
@ -1,20 +0,0 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2013-2015 Errplane Inc. |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of |
||||
this software and associated documentation files (the "Software"), to deal in |
||||
the Software without restriction, including without limitation the rights to |
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of |
||||
the Software, and to permit persons to whom the Software is furnished to do so, |
||||
subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS |
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR |
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
||||
@ -1,180 +0,0 @@ |
||||
package client |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"net/url" |
||||
"time" |
||||
|
||||
"github.com/influxdb/influxdb/tsdb" |
||||
) |
||||
|
||||
const ( |
||||
// DefaultTimeout is the default connection timeout used to connect to an InfluxDB instance
|
||||
DefaultTimeout = 0 |
||||
) |
||||
|
||||
// Config is used to specify what server to connect to.
|
||||
// URL: The URL of the server connecting to.
|
||||
// Username/Password are optional. They will be passed via basic auth if provided.
|
||||
// UserAgent: If not provided, will default "InfluxDBClient",
|
||||
// Timeout: If not provided, will default to 0 (no timeout)
|
||||
type Config struct { |
||||
URL url.URL |
||||
Username string |
||||
Password string |
||||
UserAgent string |
||||
Timeout time.Duration |
||||
Precision string |
||||
} |
||||
|
||||
// NewConfig will create a config to be used in connecting to the client
|
||||
func NewConfig() Config { |
||||
return Config{ |
||||
Timeout: DefaultTimeout, |
||||
} |
||||
} |
||||
|
||||
// Client is used to make calls to the server.
|
||||
type Client struct { |
||||
url url.URL |
||||
username string |
||||
password string |
||||
httpClient *http.Client |
||||
userAgent string |
||||
precision string |
||||
} |
||||
|
||||
const ( |
||||
ConsistencyOne = "one" |
||||
ConsistencyAll = "all" |
||||
ConsistencyQuorum = "quorum" |
||||
ConsistencyAny = "any" |
||||
) |
||||
|
||||
// NewClient will instantiate and return a connected client to issue commands to the server.
|
||||
func NewClient(c Config) (*Client, error) { |
||||
client := Client{ |
||||
url: c.URL, |
||||
username: c.Username, |
||||
password: c.Password, |
||||
httpClient: &http.Client{Timeout: c.Timeout}, |
||||
userAgent: c.UserAgent, |
||||
precision: c.Precision, |
||||
} |
||||
if client.userAgent == "" { |
||||
client.userAgent = "InfluxDBClient" |
||||
} |
||||
return &client, nil |
||||
} |
||||
|
||||
// Write takes BatchPoints and allows for writing of multiple points with defaults
|
||||
// If successful, error is nil and Response is nil
|
||||
// If an error occurs, Response may contain additional information if populated.
|
||||
func (c *Client) Write(bp BatchPoints) (*Response, error) { |
||||
u := c.url |
||||
u.Path = "write" |
||||
|
||||
var b bytes.Buffer |
||||
for _, p := range bp.Points { |
||||
if p.Raw != "" { |
||||
if _, err := b.WriteString(p.Raw); err != nil { |
||||
return nil, err |
||||
} |
||||
} else { |
||||
for k, v := range bp.Tags { |
||||
if p.Tags == nil { |
||||
p.Tags = make(map[string]string, len(bp.Tags)) |
||||
} |
||||
p.Tags[k] = v |
||||
} |
||||
|
||||
if _, err := b.WriteString(p.MarshalString()); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if err := b.WriteByte('\n'); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
req, err := http.NewRequest("POST", u.String(), &b) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
req.Header.Set("Content-Type", "") |
||||
req.Header.Set("User-Agent", c.userAgent) |
||||
if c.username != "" { |
||||
req.SetBasicAuth(c.username, c.password) |
||||
} |
||||
params := req.URL.Query() |
||||
params.Set("db", bp.Database) |
||||
params.Set("rp", bp.RetentionPolicy) |
||||
params.Set("precision", bp.Precision) |
||||
params.Set("consistency", bp.WriteConsistency) |
||||
req.URL.RawQuery = params.Encode() |
||||
|
||||
resp, err := c.httpClient.Do(req) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
defer resp.Body.Close() |
||||
|
||||
var response Response |
||||
body, err := ioutil.ReadAll(resp.Body) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { |
||||
var err = fmt.Errorf(string(body)) |
||||
response.Err = err |
||||
return &response, err |
||||
} |
||||
|
||||
return nil, nil |
||||
} |
||||
|
||||
// Structs
|
||||
|
||||
// Response represents a list of statement results.
|
||||
type Response struct { |
||||
Err error |
||||
} |
||||
|
||||
// Point defines the fields that will be written to the database
|
||||
// Measurement, Time, and Fields are required
|
||||
// Precision can be specified if the time is in epoch format (integer).
|
||||
// Valid values for Precision are n, u, ms, s, m, and h
|
||||
type Point struct { |
||||
Measurement string |
||||
Tags map[string]string |
||||
Time time.Time |
||||
Fields map[string]interface{} |
||||
Precision string |
||||
Raw string |
||||
} |
||||
|
||||
func (p *Point) MarshalString() string { |
||||
return tsdb.NewPoint(p.Measurement, p.Tags, p.Fields, p.Time).String() |
||||
} |
||||
|
||||
// BatchPoints is used to send batched data in a single write.
|
||||
// Database and Points are required
|
||||
// If no retention policy is specified, it will use the databases default retention policy.
|
||||
// If tags are specified, they will be "merged" with all points. If a point already has that tag, it is ignored.
|
||||
// If time is specified, it will be applied to any point with an empty time.
|
||||
// Precision can be specified if the time is in epoch format (integer).
|
||||
// Valid values for Precision are n, u, ms, s, m, and h
|
||||
type BatchPoints struct { |
||||
Points []Point `json:"points,omitempty"` |
||||
Database string `json:"database,omitempty"` |
||||
RetentionPolicy string `json:"retentionPolicy,omitempty"` |
||||
Tags map[string]string `json:"tags,omitempty"` |
||||
Time time.Time `json:"time,omitempty"` |
||||
Precision string `json:"precision,omitempty"` |
||||
WriteConsistency string `json:"-"` |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,14 +0,0 @@ |
||||
Copyright (c) 2013 Vaughan Newton |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated |
||||
documentation files (the "Software"), to deal in the Software without restriction, including without limitation the |
||||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit |
||||
persons to whom the Software is furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the |
||||
Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE |
||||
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR |
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
||||
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
||||
@ -1,70 +0,0 @@ |
||||
go-ini |
||||
====== |
||||
|
||||
INI parsing library for Go (golang). |
||||
|
||||
View the API documentation [here](http://godoc.org/github.com/vaughan0/go-ini). |
||||
|
||||
Usage |
||||
----- |
||||
|
||||
Parse an INI file: |
||||
|
||||
```go |
||||
import "github.com/vaughan0/go-ini" |
||||
|
||||
file, err := ini.LoadFile("myfile.ini") |
||||
``` |
||||
|
||||
Get data from the parsed file: |
||||
|
||||
```go |
||||
name, ok := file.Get("person", "name") |
||||
if !ok { |
||||
panic("'name' variable missing from 'person' section") |
||||
} |
||||
``` |
||||
|
||||
Iterate through values in a section: |
||||
|
||||
```go |
||||
for key, value := range file["mysection"] { |
||||
fmt.Printf("%s => %s\n", key, value) |
||||
} |
||||
``` |
||||
|
||||
Iterate through sections in a file: |
||||
|
||||
```go |
||||
for name, section := range file { |
||||
fmt.Printf("Section name: %s\n", name) |
||||
} |
||||
``` |
||||
|
||||
File Format |
||||
----------- |
||||
|
||||
INI files are parsed by go-ini line-by-line. Each line may be one of the following: |
||||
|
||||
* A section definition: [section-name] |
||||
* A property: key = value |
||||
* A comment: #blahblah _or_ ;blahblah |
||||
* Blank. The line will be ignored. |
||||
|
||||
Properties defined before any section headers are placed in the default section, which has |
||||
the empty string as it's key. |
||||
|
||||
Example: |
||||
|
||||
```ini |
||||
# I am a comment |
||||
; So am I! |
||||
|
||||
[apples] |
||||
colour = red or green |
||||
shape = applish |
||||
|
||||
[oranges] |
||||
shape = square |
||||
colour = blue |
||||
``` |
||||
@ -1,123 +0,0 @@ |
||||
// Package ini provides functions for parsing INI configuration files.
|
||||
package ini |
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"io" |
||||
"os" |
||||
"regexp" |
||||
"strings" |
||||
) |
||||
|
||||
var ( |
||||
sectionRegex = regexp.MustCompile(`^\[(.*)\]$`) |
||||
assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) |
||||
) |
||||
|
||||
// ErrSyntax is returned when there is a syntax error in an INI file.
|
||||
type ErrSyntax struct { |
||||
Line int |
||||
Source string // The contents of the erroneous line, without leading or trailing whitespace
|
||||
} |
||||
|
||||
func (e ErrSyntax) Error() string { |
||||
return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source) |
||||
} |
||||
|
||||
// A File represents a parsed INI file.
|
||||
type File map[string]Section |
||||
|
||||
// A Section represents a single section of an INI file.
|
||||
type Section map[string]string |
||||
|
||||
// Returns a named Section. A Section will be created if one does not already exist for the given name.
|
||||
func (f File) Section(name string) Section { |
||||
section := f[name] |
||||
if section == nil { |
||||
section = make(Section) |
||||
f[name] = section |
||||
} |
||||
return section |
||||
} |
||||
|
||||
// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup.
|
||||
func (f File) Get(section, key string) (value string, ok bool) { |
||||
if s := f[section]; s != nil { |
||||
value, ok = s[key] |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Loads INI data from a reader and stores the data in the File.
|
||||
func (f File) Load(in io.Reader) (err error) { |
||||
bufin, ok := in.(*bufio.Reader) |
||||
if !ok { |
||||
bufin = bufio.NewReader(in) |
||||
} |
||||
return parseFile(bufin, f) |
||||
} |
||||
|
||||
// Loads INI data from a named file and stores the data in the File.
|
||||
func (f File) LoadFile(file string) (err error) { |
||||
in, err := os.Open(file) |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer in.Close() |
||||
return f.Load(in) |
||||
} |
||||
|
||||
func parseFile(in *bufio.Reader, file File) (err error) { |
||||
section := "" |
||||
lineNum := 0 |
||||
for done := false; !done; { |
||||
var line string |
||||
if line, err = in.ReadString('\n'); err != nil { |
||||
if err == io.EOF { |
||||
done = true |
||||
} else { |
||||
return |
||||
} |
||||
} |
||||
lineNum++ |
||||
line = strings.TrimSpace(line) |
||||
if len(line) == 0 { |
||||
// Skip blank lines
|
||||
continue |
||||
} |
||||
if line[0] == ';' || line[0] == '#' { |
||||
// Skip comments
|
||||
continue |
||||
} |
||||
|
||||
if groups := assignRegex.FindStringSubmatch(line); groups != nil { |
||||
key, val := groups[1], groups[2] |
||||
key, val = strings.TrimSpace(key), strings.TrimSpace(val) |
||||
file.Section(section)[key] = val |
||||
} else if groups := sectionRegex.FindStringSubmatch(line); groups != nil { |
||||
name := strings.TrimSpace(groups[1]) |
||||
section = name |
||||
// Create the section if it does not exist
|
||||
file.Section(section) |
||||
} else { |
||||
return ErrSyntax{lineNum, line} |
||||
} |
||||
|
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Loads and returns a File from a reader.
|
||||
func Load(in io.Reader) (File, error) { |
||||
file := make(File) |
||||
err := file.Load(in) |
||||
return file, err |
||||
} |
||||
|
||||
// Loads and returns an INI File from a file on disk.
|
||||
func LoadFile(filename string) (File, error) { |
||||
file := make(File) |
||||
err := file.LoadFile(filename) |
||||
return file, err |
||||
} |
||||
@ -1,2 +0,0 @@ |
||||
[default] |
||||
stuff = things |
||||
@ -1,525 +0,0 @@ |
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package timeseries implements a time series structure for stats collection.
|
||||
package timeseries // import "golang.org/x/net/internal/timeseries"
|
||||
|
||||
import ( |
||||
"fmt" |
||||
"log" |
||||
"time" |
||||
) |
||||
|
||||
const ( |
||||
timeSeriesNumBuckets = 64 |
||||
minuteHourSeriesNumBuckets = 60 |
||||
) |
||||
|
||||
var timeSeriesResolutions = []time.Duration{ |
||||
1 * time.Second, |
||||
10 * time.Second, |
||||
1 * time.Minute, |
||||
10 * time.Minute, |
||||
1 * time.Hour, |
||||
6 * time.Hour, |
||||
24 * time.Hour, // 1 day
|
||||
7 * 24 * time.Hour, // 1 week
|
||||
4 * 7 * 24 * time.Hour, // 4 weeks
|
||||
16 * 7 * 24 * time.Hour, // 16 weeks
|
||||
} |
||||
|
||||
var minuteHourSeriesResolutions = []time.Duration{ |
||||
1 * time.Second, |
||||
1 * time.Minute, |
||||
} |
||||
|
||||
// An Observable is a kind of data that can be aggregated in a time series.
|
||||
type Observable interface { |
||||
Multiply(ratio float64) // Multiplies the data in self by a given ratio
|
||||
Add(other Observable) // Adds the data from a different observation to self
|
||||
Clear() // Clears the observation so it can be reused.
|
||||
CopyFrom(other Observable) // Copies the contents of a given observation to self
|
||||
} |
||||
|
||||
// Float attaches the methods of Observable to a float64.
|
||||
type Float float64 |
||||
|
||||
// NewFloat returns a Float.
|
||||
func NewFloat() Observable { |
||||
f := Float(0) |
||||
return &f |
||||
} |
||||
|
||||
// String returns the float as a string.
|
||||
func (f *Float) String() string { return fmt.Sprintf("%g", f.Value()) } |
||||
|
||||
// Value returns the float's value.
|
||||
func (f *Float) Value() float64 { return float64(*f) } |
||||
|
||||
func (f *Float) Multiply(ratio float64) { *f *= Float(ratio) } |
||||
|
||||
func (f *Float) Add(other Observable) { |
||||
o := other.(*Float) |
||||
*f += *o |
||||
} |
||||
|
||||
func (f *Float) Clear() { *f = 0 } |
||||
|
||||
func (f *Float) CopyFrom(other Observable) { |
||||
o := other.(*Float) |
||||
*f = *o |
||||
} |
||||
|
||||
// A Clock tells the current time.
|
||||
type Clock interface { |
||||
Time() time.Time |
||||
} |
||||
|
||||
type defaultClock int |
||||
|
||||
var defaultClockInstance defaultClock |
||||
|
||||
func (defaultClock) Time() time.Time { return time.Now() } |
||||
|
||||
// Information kept per level. Each level consists of a circular list of
|
||||
// observations. The start of the level may be derived from end and the
|
||||
// len(buckets) * sizeInMillis.
|
||||
type tsLevel struct { |
||||
oldest int // index to oldest bucketed Observable
|
||||
newest int // index to newest bucketed Observable
|
||||
end time.Time // end timestamp for this level
|
||||
size time.Duration // duration of the bucketed Observable
|
||||
buckets []Observable // collections of observations
|
||||
provider func() Observable // used for creating new Observable
|
||||
} |
||||
|
||||
func (l *tsLevel) Clear() { |
||||
l.oldest = 0 |
||||
l.newest = len(l.buckets) - 1 |
||||
l.end = time.Time{} |
||||
for i := range l.buckets { |
||||
if l.buckets[i] != nil { |
||||
l.buckets[i].Clear() |
||||
l.buckets[i] = nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (l *tsLevel) InitLevel(size time.Duration, numBuckets int, f func() Observable) { |
||||
l.size = size |
||||
l.provider = f |
||||
l.buckets = make([]Observable, numBuckets) |
||||
} |
||||
|
||||
// Keeps a sequence of levels. Each level is responsible for storing data at
|
||||
// a given resolution. For example, the first level stores data at a one
|
||||
// minute resolution while the second level stores data at a one hour
|
||||
// resolution.
|
||||
|
||||
// Each level is represented by a sequence of buckets. Each bucket spans an
|
||||
// interval equal to the resolution of the level. New observations are added
|
||||
// to the last bucket.
|
||||
type timeSeries struct { |
||||
provider func() Observable // make more Observable
|
||||
numBuckets int // number of buckets in each level
|
||||
levels []*tsLevel // levels of bucketed Observable
|
||||
lastAdd time.Time // time of last Observable tracked
|
||||
total Observable // convenient aggregation of all Observable
|
||||
clock Clock // Clock for getting current time
|
||||
pending Observable // observations not yet bucketed
|
||||
pendingTime time.Time // what time are we keeping in pending
|
||||
dirty bool // if there are pending observations
|
||||
} |
||||
|
||||
// init initializes a level according to the supplied criteria.
|
||||
func (ts *timeSeries) init(resolutions []time.Duration, f func() Observable, numBuckets int, clock Clock) { |
||||
ts.provider = f |
||||
ts.numBuckets = numBuckets |
||||
ts.clock = clock |
||||
ts.levels = make([]*tsLevel, len(resolutions)) |
||||
|
||||
for i := range resolutions { |
||||
if i > 0 && resolutions[i-1] >= resolutions[i] { |
||||
log.Print("timeseries: resolutions must be monotonically increasing") |
||||
break |
||||
} |
||||
newLevel := new(tsLevel) |
||||
newLevel.InitLevel(resolutions[i], ts.numBuckets, ts.provider) |
||||
ts.levels[i] = newLevel |
||||
} |
||||
|
||||
ts.Clear() |
||||
} |
||||
|
||||
// Clear removes all observations from the time series.
|
||||
func (ts *timeSeries) Clear() { |
||||
ts.lastAdd = time.Time{} |
||||
ts.total = ts.resetObservation(ts.total) |
||||
ts.pending = ts.resetObservation(ts.pending) |
||||
ts.pendingTime = time.Time{} |
||||
ts.dirty = false |
||||
|
||||
for i := range ts.levels { |
||||
ts.levels[i].Clear() |
||||
} |
||||
} |
||||
|
||||
// Add records an observation at the current time.
|
||||
func (ts *timeSeries) Add(observation Observable) { |
||||
ts.AddWithTime(observation, ts.clock.Time()) |
||||
} |
||||
|
||||
// AddWithTime records an observation at the specified time.
|
||||
func (ts *timeSeries) AddWithTime(observation Observable, t time.Time) { |
||||
|
||||
smallBucketDuration := ts.levels[0].size |
||||
|
||||
if t.After(ts.lastAdd) { |
||||
ts.lastAdd = t |
||||
} |
||||
|
||||
if t.After(ts.pendingTime) { |
||||
ts.advance(t) |
||||
ts.mergePendingUpdates() |
||||
ts.pendingTime = ts.levels[0].end |
||||
ts.pending.CopyFrom(observation) |
||||
ts.dirty = true |
||||
} else if t.After(ts.pendingTime.Add(-1 * smallBucketDuration)) { |
||||
// The observation is close enough to go into the pending bucket.
|
||||
// This compensates for clock skewing and small scheduling delays
|
||||
// by letting the update stay in the fast path.
|
||||
ts.pending.Add(observation) |
||||
ts.dirty = true |
||||
} else { |
||||
ts.mergeValue(observation, t) |
||||
} |
||||
} |
||||
|
||||
// mergeValue inserts the observation at the specified time in the past into all levels.
|
||||
func (ts *timeSeries) mergeValue(observation Observable, t time.Time) { |
||||
for _, level := range ts.levels { |
||||
index := (ts.numBuckets - 1) - int(level.end.Sub(t)/level.size) |
||||
if 0 <= index && index < ts.numBuckets { |
||||
bucketNumber := (level.oldest + index) % ts.numBuckets |
||||
if level.buckets[bucketNumber] == nil { |
||||
level.buckets[bucketNumber] = level.provider() |
||||
} |
||||
level.buckets[bucketNumber].Add(observation) |
||||
} |
||||
} |
||||
ts.total.Add(observation) |
||||
} |
||||
|
||||
// mergePendingUpdates applies the pending updates into all levels.
|
||||
func (ts *timeSeries) mergePendingUpdates() { |
||||
if ts.dirty { |
||||
ts.mergeValue(ts.pending, ts.pendingTime) |
||||
ts.pending = ts.resetObservation(ts.pending) |
||||
ts.dirty = false |
||||
} |
||||
} |
||||
|
||||
// advance cycles the buckets at each level until the latest bucket in
|
||||
// each level can hold the time specified.
|
||||
func (ts *timeSeries) advance(t time.Time) { |
||||
if !t.After(ts.levels[0].end) { |
||||
return |
||||
} |
||||
for i := 0; i < len(ts.levels); i++ { |
||||
level := ts.levels[i] |
||||
if !level.end.Before(t) { |
||||
break |
||||
} |
||||
|
||||
// If the time is sufficiently far, just clear the level and advance
|
||||
// directly.
|
||||
if !t.Before(level.end.Add(level.size * time.Duration(ts.numBuckets))) { |
||||
for _, b := range level.buckets { |
||||
ts.resetObservation(b) |
||||
} |
||||
level.end = time.Unix(0, (t.UnixNano()/level.size.Nanoseconds())*level.size.Nanoseconds()) |
||||
} |
||||
|
||||
for t.After(level.end) { |
||||
level.end = level.end.Add(level.size) |
||||
level.newest = level.oldest |
||||
level.oldest = (level.oldest + 1) % ts.numBuckets |
||||
ts.resetObservation(level.buckets[level.newest]) |
||||
} |
||||
|
||||
t = level.end |
||||
} |
||||
} |
||||
|
||||
// Latest returns the sum of the num latest buckets from the level.
|
||||
func (ts *timeSeries) Latest(level, num int) Observable { |
||||
now := ts.clock.Time() |
||||
if ts.levels[0].end.Before(now) { |
||||
ts.advance(now) |
||||
} |
||||
|
||||
ts.mergePendingUpdates() |
||||
|
||||
result := ts.provider() |
||||
l := ts.levels[level] |
||||
index := l.newest |
||||
|
||||
for i := 0; i < num; i++ { |
||||
if l.buckets[index] != nil { |
||||
result.Add(l.buckets[index]) |
||||
} |
||||
if index == 0 { |
||||
index = ts.numBuckets |
||||
} |
||||
index-- |
||||
} |
||||
|
||||
return result |
||||
} |
||||
|
||||
// LatestBuckets returns a copy of the num latest buckets from level.
|
||||
func (ts *timeSeries) LatestBuckets(level, num int) []Observable { |
||||
if level < 0 || level > len(ts.levels) { |
||||
log.Print("timeseries: bad level argument: ", level) |
||||
return nil |
||||
} |
||||
if num < 0 || num >= ts.numBuckets { |
||||
log.Print("timeseries: bad num argument: ", num) |
||||
return nil |
||||
} |
||||
|
||||
results := make([]Observable, num) |
||||
now := ts.clock.Time() |
||||
if ts.levels[0].end.Before(now) { |
||||
ts.advance(now) |
||||
} |
||||
|
||||
ts.mergePendingUpdates() |
||||
|
||||
l := ts.levels[level] |
||||
index := l.newest |
||||
|
||||
for i := 0; i < num; i++ { |
||||
result := ts.provider() |
||||
results[i] = result |
||||
if l.buckets[index] != nil { |
||||
result.CopyFrom(l.buckets[index]) |
||||
} |
||||
|
||||
if index == 0 { |
||||
index = ts.numBuckets |
||||
} |
||||
index -= 1 |
||||
} |
||||
return results |
||||
} |
||||
|
||||
// ScaleBy updates observations by scaling by factor.
|
||||
func (ts *timeSeries) ScaleBy(factor float64) { |
||||
for _, l := range ts.levels { |
||||
for i := 0; i < ts.numBuckets; i++ { |
||||
l.buckets[i].Multiply(factor) |
||||
} |
||||
} |
||||
|
||||
ts.total.Multiply(factor) |
||||
ts.pending.Multiply(factor) |
||||
} |
||||
|
||||
// Range returns the sum of observations added over the specified time range.
|
||||
// If start or finish times don't fall on bucket boundaries of the same
|
||||
// level, then return values are approximate answers.
|
||||
func (ts *timeSeries) Range(start, finish time.Time) Observable { |
||||
return ts.ComputeRange(start, finish, 1)[0] |
||||
} |
||||
|
||||
// Recent returns the sum of observations from the last delta.
|
||||
func (ts *timeSeries) Recent(delta time.Duration) Observable { |
||||
now := ts.clock.Time() |
||||
return ts.Range(now.Add(-delta), now) |
||||
} |
||||
|
||||
// Total returns the total of all observations.
|
||||
func (ts *timeSeries) Total() Observable { |
||||
ts.mergePendingUpdates() |
||||
return ts.total |
||||
} |
||||
|
||||
// ComputeRange computes a specified number of values into a slice using
|
||||
// the observations recorded over the specified time period. The return
|
||||
// values are approximate if the start or finish times don't fall on the
|
||||
// bucket boundaries at the same level or if the number of buckets spanning
|
||||
// the range is not an integral multiple of num.
|
||||
func (ts *timeSeries) ComputeRange(start, finish time.Time, num int) []Observable { |
||||
if start.After(finish) { |
||||
log.Printf("timeseries: start > finish, %v>%v", start, finish) |
||||
return nil |
||||
} |
||||
|
||||
if num < 0 { |
||||
log.Printf("timeseries: num < 0, %v", num) |
||||
return nil |
||||
} |
||||
|
||||
results := make([]Observable, num) |
||||
|
||||
for _, l := range ts.levels { |
||||
if !start.Before(l.end.Add(-l.size * time.Duration(ts.numBuckets))) { |
||||
ts.extract(l, start, finish, num, results) |
||||
return results |
||||
} |
||||
} |
||||
|
||||
// Failed to find a level that covers the desired range. So just
|
||||
// extract from the last level, even if it doesn't cover the entire
|
||||
// desired range.
|
||||
ts.extract(ts.levels[len(ts.levels)-1], start, finish, num, results) |
||||
|
||||
return results |
||||
} |
||||
|
||||
// RecentList returns the specified number of values in slice over the most
|
||||
// recent time period of the specified range.
|
||||
func (ts *timeSeries) RecentList(delta time.Duration, num int) []Observable { |
||||
if delta < 0 { |
||||
return nil |
||||
} |
||||
now := ts.clock.Time() |
||||
return ts.ComputeRange(now.Add(-delta), now, num) |
||||
} |
||||
|
||||
// extract returns a slice of specified number of observations from a given
|
||||
// level over a given range.
|
||||
func (ts *timeSeries) extract(l *tsLevel, start, finish time.Time, num int, results []Observable) { |
||||
ts.mergePendingUpdates() |
||||
|
||||
srcInterval := l.size |
||||
dstInterval := finish.Sub(start) / time.Duration(num) |
||||
dstStart := start |
||||
srcStart := l.end.Add(-srcInterval * time.Duration(ts.numBuckets)) |
||||
|
||||
srcIndex := 0 |
||||
|
||||
// Where should scanning start?
|
||||
if dstStart.After(srcStart) { |
||||
advance := dstStart.Sub(srcStart) / srcInterval |
||||
srcIndex += int(advance) |
||||
srcStart = srcStart.Add(advance * srcInterval) |
||||
} |
||||
|
||||
// The i'th value is computed as show below.
|
||||
// interval = (finish/start)/num
|
||||
// i'th value = sum of observation in range
|
||||
// [ start + i * interval,
|
||||
// start + (i + 1) * interval )
|
||||
for i := 0; i < num; i++ { |
||||
results[i] = ts.resetObservation(results[i]) |
||||
dstEnd := dstStart.Add(dstInterval) |
||||
for srcIndex < ts.numBuckets && srcStart.Before(dstEnd) { |
||||
srcEnd := srcStart.Add(srcInterval) |
||||
if srcEnd.After(ts.lastAdd) { |
||||
srcEnd = ts.lastAdd |
||||
} |
||||
|
||||
if !srcEnd.Before(dstStart) { |
||||
srcValue := l.buckets[(srcIndex+l.oldest)%ts.numBuckets] |
||||
if !srcStart.Before(dstStart) && !srcEnd.After(dstEnd) { |
||||
// dst completely contains src.
|
||||
if srcValue != nil { |
||||
results[i].Add(srcValue) |
||||
} |
||||
} else { |
||||
// dst partially overlaps src.
|
||||
overlapStart := maxTime(srcStart, dstStart) |
||||
overlapEnd := minTime(srcEnd, dstEnd) |
||||
base := srcEnd.Sub(srcStart) |
||||
fraction := overlapEnd.Sub(overlapStart).Seconds() / base.Seconds() |
||||
|
||||
used := ts.provider() |
||||
if srcValue != nil { |
||||
used.CopyFrom(srcValue) |
||||
} |
||||
used.Multiply(fraction) |
||||
results[i].Add(used) |
||||
} |
||||
|
||||
if srcEnd.After(dstEnd) { |
||||
break |
||||
} |
||||
} |
||||
srcIndex++ |
||||
srcStart = srcStart.Add(srcInterval) |
||||
} |
||||
dstStart = dstStart.Add(dstInterval) |
||||
} |
||||
} |
||||
|
||||
// resetObservation clears the content so the struct may be reused.
|
||||
func (ts *timeSeries) resetObservation(observation Observable) Observable { |
||||
if observation == nil { |
||||
observation = ts.provider() |
||||
} else { |
||||
observation.Clear() |
||||
} |
||||
return observation |
||||
} |
||||
|
||||
// TimeSeries tracks data at granularities from 1 second to 16 weeks.
|
||||
type TimeSeries struct { |
||||
timeSeries |
||||
} |
||||
|
||||
// NewTimeSeries creates a new TimeSeries using the function provided for creating new Observable.
|
||||
func NewTimeSeries(f func() Observable) *TimeSeries { |
||||
return NewTimeSeriesWithClock(f, defaultClockInstance) |
||||
} |
||||
|
||||
// NewTimeSeriesWithClock creates a new TimeSeries using the function provided for creating new Observable and the clock for
|
||||
// assigning timestamps.
|
||||
func NewTimeSeriesWithClock(f func() Observable, clock Clock) *TimeSeries { |
||||
ts := new(TimeSeries) |
||||
ts.timeSeries.init(timeSeriesResolutions, f, timeSeriesNumBuckets, clock) |
||||
return ts |
||||
} |
||||
|
||||
// MinuteHourSeries tracks data at granularities of 1 minute and 1 hour.
|
||||
type MinuteHourSeries struct { |
||||
timeSeries |
||||
} |
||||
|
||||
// NewMinuteHourSeries creates a new MinuteHourSeries using the function provided for creating new Observable.
|
||||
func NewMinuteHourSeries(f func() Observable) *MinuteHourSeries { |
||||
return NewMinuteHourSeriesWithClock(f, defaultClockInstance) |
||||
} |
||||
|
||||
// NewMinuteHourSeriesWithClock creates a new MinuteHourSeries using the function provided for creating new Observable and the clock for
|
||||
// assigning timestamps.
|
||||
func NewMinuteHourSeriesWithClock(f func() Observable, clock Clock) *MinuteHourSeries { |
||||
ts := new(MinuteHourSeries) |
||||
ts.timeSeries.init(minuteHourSeriesResolutions, f, |
||||
minuteHourSeriesNumBuckets, clock) |
||||
return ts |
||||
} |
||||
|
||||
func (ts *MinuteHourSeries) Minute() Observable { |
||||
return ts.timeSeries.Latest(0, 60) |
||||
} |
||||
|
||||
func (ts *MinuteHourSeries) Hour() Observable { |
||||
return ts.timeSeries.Latest(1, 60) |
||||
} |
||||
|
||||
func minTime(a, b time.Time) time.Time { |
||||
if a.Before(b) { |
||||
return a |
||||
} |
||||
return b |
||||
} |
||||
|
||||
func maxTime(a, b time.Time) time.Time { |
||||
if a.After(b) { |
||||
return a |
||||
} |
||||
return b |
||||
} |
||||
@ -1,355 +0,0 @@ |
||||
// Code generated by protoc-gen-go.
|
||||
// source: google.golang.org/appengine/internal/urlfetch/urlfetch_service.proto
|
||||
// DO NOT EDIT!
|
||||
|
||||
/* |
||||
Package urlfetch is a generated protocol buffer package. |
||||
|
||||
It is generated from these files: |
||||
google.golang.org/appengine/internal/urlfetch/urlfetch_service.proto |
||||
|
||||
It has these top-level messages: |
||||
URLFetchServiceError |
||||
URLFetchRequest |
||||
URLFetchResponse |
||||
*/ |
||||
package urlfetch |
||||
|
||||
import proto "github.com/golang/protobuf/proto" |
||||
import fmt "fmt" |
||||
import math "math" |
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
var _ = proto.Marshal |
||||
var _ = fmt.Errorf |
||||
var _ = math.Inf |
||||
|
||||
type URLFetchServiceError_ErrorCode int32 |
||||
|
||||
const ( |
||||
URLFetchServiceError_OK URLFetchServiceError_ErrorCode = 0 |
||||
URLFetchServiceError_INVALID_URL URLFetchServiceError_ErrorCode = 1 |
||||
URLFetchServiceError_FETCH_ERROR URLFetchServiceError_ErrorCode = 2 |
||||
URLFetchServiceError_UNSPECIFIED_ERROR URLFetchServiceError_ErrorCode = 3 |
||||
URLFetchServiceError_RESPONSE_TOO_LARGE URLFetchServiceError_ErrorCode = 4 |
||||
URLFetchServiceError_DEADLINE_EXCEEDED URLFetchServiceError_ErrorCode = 5 |
||||
URLFetchServiceError_SSL_CERTIFICATE_ERROR URLFetchServiceError_ErrorCode = 6 |
||||
URLFetchServiceError_DNS_ERROR URLFetchServiceError_ErrorCode = 7 |
||||
URLFetchServiceError_CLOSED URLFetchServiceError_ErrorCode = 8 |
||||
URLFetchServiceError_INTERNAL_TRANSIENT_ERROR URLFetchServiceError_ErrorCode = 9 |
||||
URLFetchServiceError_TOO_MANY_REDIRECTS URLFetchServiceError_ErrorCode = 10 |
||||
URLFetchServiceError_MALFORMED_REPLY URLFetchServiceError_ErrorCode = 11 |
||||
URLFetchServiceError_CONNECTION_ERROR URLFetchServiceError_ErrorCode = 12 |
||||
) |
||||
|
||||
var URLFetchServiceError_ErrorCode_name = map[int32]string{ |
||||
0: "OK", |
||||
1: "INVALID_URL", |
||||
2: "FETCH_ERROR", |
||||
3: "UNSPECIFIED_ERROR", |
||||
4: "RESPONSE_TOO_LARGE", |
||||
5: "DEADLINE_EXCEEDED", |
||||
6: "SSL_CERTIFICATE_ERROR", |
||||
7: "DNS_ERROR", |
||||
8: "CLOSED", |
||||
9: "INTERNAL_TRANSIENT_ERROR", |
||||
10: "TOO_MANY_REDIRECTS", |
||||
11: "MALFORMED_REPLY", |
||||
12: "CONNECTION_ERROR", |
||||
} |
||||
var URLFetchServiceError_ErrorCode_value = map[string]int32{ |
||||
"OK": 0, |
||||
"INVALID_URL": 1, |
||||
"FETCH_ERROR": 2, |
||||
"UNSPECIFIED_ERROR": 3, |
||||
"RESPONSE_TOO_LARGE": 4, |
||||
"DEADLINE_EXCEEDED": 5, |
||||
"SSL_CERTIFICATE_ERROR": 6, |
||||
"DNS_ERROR": 7, |
||||
"CLOSED": 8, |
||||
"INTERNAL_TRANSIENT_ERROR": 9, |
||||
"TOO_MANY_REDIRECTS": 10, |
||||
"MALFORMED_REPLY": 11, |
||||
"CONNECTION_ERROR": 12, |
||||
} |
||||
|
||||
func (x URLFetchServiceError_ErrorCode) Enum() *URLFetchServiceError_ErrorCode { |
||||
p := new(URLFetchServiceError_ErrorCode) |
||||
*p = x |
||||
return p |
||||
} |
||||
func (x URLFetchServiceError_ErrorCode) String() string { |
||||
return proto.EnumName(URLFetchServiceError_ErrorCode_name, int32(x)) |
||||
} |
||||
func (x *URLFetchServiceError_ErrorCode) UnmarshalJSON(data []byte) error { |
||||
value, err := proto.UnmarshalJSONEnum(URLFetchServiceError_ErrorCode_value, data, "URLFetchServiceError_ErrorCode") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
*x = URLFetchServiceError_ErrorCode(value) |
||||
return nil |
||||
} |
||||
|
||||
type URLFetchRequest_RequestMethod int32 |
||||
|
||||
const ( |
||||
URLFetchRequest_GET URLFetchRequest_RequestMethod = 1 |
||||
URLFetchRequest_POST URLFetchRequest_RequestMethod = 2 |
||||
URLFetchRequest_HEAD URLFetchRequest_RequestMethod = 3 |
||||
URLFetchRequest_PUT URLFetchRequest_RequestMethod = 4 |
||||
URLFetchRequest_DELETE URLFetchRequest_RequestMethod = 5 |
||||
URLFetchRequest_PATCH URLFetchRequest_RequestMethod = 6 |
||||
) |
||||
|
||||
var URLFetchRequest_RequestMethod_name = map[int32]string{ |
||||
1: "GET", |
||||
2: "POST", |
||||
3: "HEAD", |
||||
4: "PUT", |
||||
5: "DELETE", |
||||
6: "PATCH", |
||||
} |
||||
var URLFetchRequest_RequestMethod_value = map[string]int32{ |
||||
"GET": 1, |
||||
"POST": 2, |
||||
"HEAD": 3, |
||||
"PUT": 4, |
||||
"DELETE": 5, |
||||
"PATCH": 6, |
||||
} |
||||
|
||||
func (x URLFetchRequest_RequestMethod) Enum() *URLFetchRequest_RequestMethod { |
||||
p := new(URLFetchRequest_RequestMethod) |
||||
*p = x |
||||
return p |
||||
} |
||||
func (x URLFetchRequest_RequestMethod) String() string { |
||||
return proto.EnumName(URLFetchRequest_RequestMethod_name, int32(x)) |
||||
} |
||||
func (x *URLFetchRequest_RequestMethod) UnmarshalJSON(data []byte) error { |
||||
value, err := proto.UnmarshalJSONEnum(URLFetchRequest_RequestMethod_value, data, "URLFetchRequest_RequestMethod") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
*x = URLFetchRequest_RequestMethod(value) |
||||
return nil |
||||
} |
||||
|
||||
type URLFetchServiceError struct { |
||||
XXX_unrecognized []byte `json:"-"` |
||||
} |
||||
|
||||
func (m *URLFetchServiceError) Reset() { *m = URLFetchServiceError{} } |
||||
func (m *URLFetchServiceError) String() string { return proto.CompactTextString(m) } |
||||
func (*URLFetchServiceError) ProtoMessage() {} |
||||
|
||||
type URLFetchRequest struct { |
||||
Method *URLFetchRequest_RequestMethod `protobuf:"varint,1,req,name=Method,enum=appengine.URLFetchRequest_RequestMethod" json:"Method,omitempty"` |
||||
Url *string `protobuf:"bytes,2,req,name=Url" json:"Url,omitempty"` |
||||
Header []*URLFetchRequest_Header `protobuf:"group,3,rep,name=Header" json:"header,omitempty"` |
||||
Payload []byte `protobuf:"bytes,6,opt,name=Payload" json:"Payload,omitempty"` |
||||
FollowRedirects *bool `protobuf:"varint,7,opt,name=FollowRedirects,def=1" json:"FollowRedirects,omitempty"` |
||||
Deadline *float64 `protobuf:"fixed64,8,opt,name=Deadline" json:"Deadline,omitempty"` |
||||
MustValidateServerCertificate *bool `protobuf:"varint,9,opt,name=MustValidateServerCertificate,def=1" json:"MustValidateServerCertificate,omitempty"` |
||||
XXX_unrecognized []byte `json:"-"` |
||||
} |
||||
|
||||
func (m *URLFetchRequest) Reset() { *m = URLFetchRequest{} } |
||||
func (m *URLFetchRequest) String() string { return proto.CompactTextString(m) } |
||||
func (*URLFetchRequest) ProtoMessage() {} |
||||
|
||||
const Default_URLFetchRequest_FollowRedirects bool = true |
||||
const Default_URLFetchRequest_MustValidateServerCertificate bool = true |
||||
|
||||
func (m *URLFetchRequest) GetMethod() URLFetchRequest_RequestMethod { |
||||
if m != nil && m.Method != nil { |
||||
return *m.Method |
||||
} |
||||
return URLFetchRequest_GET |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetUrl() string { |
||||
if m != nil && m.Url != nil { |
||||
return *m.Url |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetHeader() []*URLFetchRequest_Header { |
||||
if m != nil { |
||||
return m.Header |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetPayload() []byte { |
||||
if m != nil { |
||||
return m.Payload |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetFollowRedirects() bool { |
||||
if m != nil && m.FollowRedirects != nil { |
||||
return *m.FollowRedirects |
||||
} |
||||
return Default_URLFetchRequest_FollowRedirects |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetDeadline() float64 { |
||||
if m != nil && m.Deadline != nil { |
||||
return *m.Deadline |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (m *URLFetchRequest) GetMustValidateServerCertificate() bool { |
||||
if m != nil && m.MustValidateServerCertificate != nil { |
||||
return *m.MustValidateServerCertificate |
||||
} |
||||
return Default_URLFetchRequest_MustValidateServerCertificate |
||||
} |
||||
|
||||
type URLFetchRequest_Header struct { |
||||
Key *string `protobuf:"bytes,4,req,name=Key" json:"Key,omitempty"` |
||||
Value *string `protobuf:"bytes,5,req,name=Value" json:"Value,omitempty"` |
||||
XXX_unrecognized []byte `json:"-"` |
||||
} |
||||
|
||||
func (m *URLFetchRequest_Header) Reset() { *m = URLFetchRequest_Header{} } |
||||
func (m *URLFetchRequest_Header) String() string { return proto.CompactTextString(m) } |
||||
func (*URLFetchRequest_Header) ProtoMessage() {} |
||||
|
||||
func (m *URLFetchRequest_Header) GetKey() string { |
||||
if m != nil && m.Key != nil { |
||||
return *m.Key |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (m *URLFetchRequest_Header) GetValue() string { |
||||
if m != nil && m.Value != nil { |
||||
return *m.Value |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
type URLFetchResponse struct { |
||||
Content []byte `protobuf:"bytes,1,opt,name=Content" json:"Content,omitempty"` |
||||
StatusCode *int32 `protobuf:"varint,2,req,name=StatusCode" json:"StatusCode,omitempty"` |
||||
Header []*URLFetchResponse_Header `protobuf:"group,3,rep,name=Header" json:"header,omitempty"` |
||||
ContentWasTruncated *bool `protobuf:"varint,6,opt,name=ContentWasTruncated,def=0" json:"ContentWasTruncated,omitempty"` |
||||
ExternalBytesSent *int64 `protobuf:"varint,7,opt,name=ExternalBytesSent" json:"ExternalBytesSent,omitempty"` |
||||
ExternalBytesReceived *int64 `protobuf:"varint,8,opt,name=ExternalBytesReceived" json:"ExternalBytesReceived,omitempty"` |
||||
FinalUrl *string `protobuf:"bytes,9,opt,name=FinalUrl" json:"FinalUrl,omitempty"` |
||||
ApiCpuMilliseconds *int64 `protobuf:"varint,10,opt,name=ApiCpuMilliseconds,def=0" json:"ApiCpuMilliseconds,omitempty"` |
||||
ApiBytesSent *int64 `protobuf:"varint,11,opt,name=ApiBytesSent,def=0" json:"ApiBytesSent,omitempty"` |
||||
ApiBytesReceived *int64 `protobuf:"varint,12,opt,name=ApiBytesReceived,def=0" json:"ApiBytesReceived,omitempty"` |
||||
XXX_unrecognized []byte `json:"-"` |
||||
} |
||||
|
||||
func (m *URLFetchResponse) Reset() { *m = URLFetchResponse{} } |
||||
func (m *URLFetchResponse) String() string { return proto.CompactTextString(m) } |
||||
func (*URLFetchResponse) ProtoMessage() {} |
||||
|
||||
const Default_URLFetchResponse_ContentWasTruncated bool = false |
||||
const Default_URLFetchResponse_ApiCpuMilliseconds int64 = 0 |
||||
const Default_URLFetchResponse_ApiBytesSent int64 = 0 |
||||
const Default_URLFetchResponse_ApiBytesReceived int64 = 0 |
||||
|
||||
func (m *URLFetchResponse) GetContent() []byte { |
||||
if m != nil { |
||||
return m.Content |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetStatusCode() int32 { |
||||
if m != nil && m.StatusCode != nil { |
||||
return *m.StatusCode |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetHeader() []*URLFetchResponse_Header { |
||||
if m != nil { |
||||
return m.Header |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetContentWasTruncated() bool { |
||||
if m != nil && m.ContentWasTruncated != nil { |
||||
return *m.ContentWasTruncated |
||||
} |
||||
return Default_URLFetchResponse_ContentWasTruncated |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetExternalBytesSent() int64 { |
||||
if m != nil && m.ExternalBytesSent != nil { |
||||
return *m.ExternalBytesSent |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetExternalBytesReceived() int64 { |
||||
if m != nil && m.ExternalBytesReceived != nil { |
||||
return *m.ExternalBytesReceived |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetFinalUrl() string { |
||||
if m != nil && m.FinalUrl != nil { |
||||
return *m.FinalUrl |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetApiCpuMilliseconds() int64 { |
||||
if m != nil && m.ApiCpuMilliseconds != nil { |
||||
return *m.ApiCpuMilliseconds |
||||
} |
||||
return Default_URLFetchResponse_ApiCpuMilliseconds |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetApiBytesSent() int64 { |
||||
if m != nil && m.ApiBytesSent != nil { |
||||
return *m.ApiBytesSent |
||||
} |
||||
return Default_URLFetchResponse_ApiBytesSent |
||||
} |
||||
|
||||
func (m *URLFetchResponse) GetApiBytesReceived() int64 { |
||||
if m != nil && m.ApiBytesReceived != nil { |
||||
return *m.ApiBytesReceived |
||||
} |
||||
return Default_URLFetchResponse_ApiBytesReceived |
||||
} |
||||
|
||||
type URLFetchResponse_Header struct { |
||||
Key *string `protobuf:"bytes,4,req,name=Key" json:"Key,omitempty"` |
||||
Value *string `protobuf:"bytes,5,req,name=Value" json:"Value,omitempty"` |
||||
XXX_unrecognized []byte `json:"-"` |
||||
} |
||||
|
||||
func (m *URLFetchResponse_Header) Reset() { *m = URLFetchResponse_Header{} } |
||||
func (m *URLFetchResponse_Header) String() string { return proto.CompactTextString(m) } |
||||
func (*URLFetchResponse_Header) ProtoMessage() {} |
||||
|
||||
func (m *URLFetchResponse_Header) GetKey() string { |
||||
if m != nil && m.Key != nil { |
||||
return *m.Key |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (m *URLFetchResponse_Header) GetValue() string { |
||||
if m != nil && m.Value != nil { |
||||
return *m.Value |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func init() { |
||||
} |
||||
@ -1,64 +0,0 @@ |
||||
syntax = "proto2"; |
||||
option go_package = "urlfetch"; |
||||
|
||||
package appengine; |
||||
|
||||
message URLFetchServiceError { |
||||
enum ErrorCode { |
||||
OK = 0; |
||||
INVALID_URL = 1; |
||||
FETCH_ERROR = 2; |
||||
UNSPECIFIED_ERROR = 3; |
||||
RESPONSE_TOO_LARGE = 4; |
||||
DEADLINE_EXCEEDED = 5; |
||||
SSL_CERTIFICATE_ERROR = 6; |
||||
DNS_ERROR = 7; |
||||
CLOSED = 8; |
||||
INTERNAL_TRANSIENT_ERROR = 9; |
||||
TOO_MANY_REDIRECTS = 10; |
||||
MALFORMED_REPLY = 11; |
||||
CONNECTION_ERROR = 12; |
||||
} |
||||
} |
||||
|
||||
message URLFetchRequest { |
||||
enum RequestMethod { |
||||
GET = 1; |
||||
POST = 2; |
||||
HEAD = 3; |
||||
PUT = 4; |
||||
DELETE = 5; |
||||
PATCH = 6; |
||||
} |
||||
required RequestMethod Method = 1; |
||||
required string Url = 2; |
||||
repeated group Header = 3 { |
||||
required string Key = 4; |
||||
required string Value = 5; |
||||
} |
||||
optional bytes Payload = 6 [ctype=CORD]; |
||||
|
||||
optional bool FollowRedirects = 7 [default=true]; |
||||
|
||||
optional double Deadline = 8; |
||||
|
||||
optional bool MustValidateServerCertificate = 9 [default=true]; |
||||
} |
||||
|
||||
message URLFetchResponse { |
||||
optional bytes Content = 1; |
||||
required int32 StatusCode = 2; |
||||
repeated group Header = 3 { |
||||
required string Key = 4; |
||||
required string Value = 5; |
||||
} |
||||
optional bool ContentWasTruncated = 6 [default=false]; |
||||
optional int64 ExternalBytesSent = 7; |
||||
optional int64 ExternalBytesReceived = 8; |
||||
|
||||
optional string FinalUrl = 9; |
||||
|
||||
optional int64 ApiCpuMilliseconds = 10 [default=0]; |
||||
optional int64 ApiBytesSent = 11 [default=0]; |
||||
optional int64 ApiBytesReceived = 12 [default=0]; |
||||
} |
||||
@ -1,210 +0,0 @@ |
||||
// Copyright 2011 Google Inc. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package urlfetch provides an http.RoundTripper implementation
|
||||
// for fetching URLs via App Engine's urlfetch service.
|
||||
package urlfetch // import "google.golang.org/appengine/urlfetch"
|
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"net/url" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/golang/protobuf/proto" |
||||
"golang.org/x/net/context" |
||||
|
||||
"google.golang.org/appengine/internal" |
||||
pb "google.golang.org/appengine/internal/urlfetch" |
||||
) |
||||
|
||||
// Transport is an implementation of http.RoundTripper for
|
||||
// App Engine. Users should generally create an http.Client using
|
||||
// this transport and use the Client rather than using this transport
|
||||
// directly.
|
||||
type Transport struct { |
||||
Context context.Context |
||||
|
||||
// Controls whether the application checks the validity of SSL certificates
|
||||
// over HTTPS connections. A value of false (the default) instructs the
|
||||
// application to send a request to the server only if the certificate is
|
||||
// valid and signed by a trusted certificate authority (CA), and also
|
||||
// includes a hostname that matches the certificate. A value of true
|
||||
// instructs the application to perform no certificate validation.
|
||||
AllowInvalidServerCertificate bool |
||||
} |
||||
|
||||
// Verify statically that *Transport implements http.RoundTripper.
|
||||
var _ http.RoundTripper = (*Transport)(nil) |
||||
|
||||
// Client returns an *http.Client using a default urlfetch Transport. This
|
||||
// client will have the default deadline of 5 seconds, and will check the
|
||||
// validity of SSL certificates.
|
||||
//
|
||||
// Any deadline of the provided context will be used for requests through this client;
|
||||
// if the client does not have a deadline then a 5 second default is used.
|
||||
func Client(ctx context.Context) *http.Client { |
||||
return &http.Client{ |
||||
Transport: &Transport{ |
||||
Context: ctx, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
type bodyReader struct { |
||||
content []byte |
||||
truncated bool |
||||
closed bool |
||||
} |
||||
|
||||
// ErrTruncatedBody is the error returned after the final Read() from a
|
||||
// response's Body if the body has been truncated by App Engine's proxy.
|
||||
var ErrTruncatedBody = errors.New("urlfetch: truncated body") |
||||
|
||||
func statusCodeToText(code int) string { |
||||
if t := http.StatusText(code); t != "" { |
||||
return t |
||||
} |
||||
return strconv.Itoa(code) |
||||
} |
||||
|
||||
func (br *bodyReader) Read(p []byte) (n int, err error) { |
||||
if br.closed { |
||||
if br.truncated { |
||||
return 0, ErrTruncatedBody |
||||
} |
||||
return 0, io.EOF |
||||
} |
||||
n = copy(p, br.content) |
||||
if n > 0 { |
||||
br.content = br.content[n:] |
||||
return |
||||
} |
||||
if br.truncated { |
||||
br.closed = true |
||||
return 0, ErrTruncatedBody |
||||
} |
||||
return 0, io.EOF |
||||
} |
||||
|
||||
func (br *bodyReader) Close() error { |
||||
br.closed = true |
||||
br.content = nil |
||||
return nil |
||||
} |
||||
|
||||
// A map of the URL Fetch-accepted methods that take a request body.
|
||||
var methodAcceptsRequestBody = map[string]bool{ |
||||
"POST": true, |
||||
"PUT": true, |
||||
"PATCH": true, |
||||
} |
||||
|
||||
// urlString returns a valid string given a URL. This function is necessary because
|
||||
// the String method of URL doesn't correctly handle URLs with non-empty Opaque values.
|
||||
// See http://code.google.com/p/go/issues/detail?id=4860.
|
||||
func urlString(u *url.URL) string { |
||||
if u.Opaque == "" || strings.HasPrefix(u.Opaque, "//") { |
||||
return u.String() |
||||
} |
||||
aux := *u |
||||
aux.Opaque = "//" + aux.Host + aux.Opaque |
||||
return aux.String() |
||||
} |
||||
|
||||
// RoundTrip issues a single HTTP request and returns its response. Per the
|
||||
// http.RoundTripper interface, RoundTrip only returns an error if there
|
||||
// was an unsupported request or the URL Fetch proxy fails.
|
||||
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
|
||||
// errors as far as the transport is concerned and will be returned
|
||||
// with err set to nil.
|
||||
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) { |
||||
methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method] |
||||
if !ok { |
||||
return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method) |
||||
} |
||||
|
||||
method := pb.URLFetchRequest_RequestMethod(methNum) |
||||
|
||||
freq := &pb.URLFetchRequest{ |
||||
Method: &method, |
||||
Url: proto.String(urlString(req.URL)), |
||||
FollowRedirects: proto.Bool(false), // http.Client's responsibility
|
||||
MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate), |
||||
} |
||||
if deadline, ok := t.Context.Deadline(); ok { |
||||
freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds()) |
||||
} |
||||
|
||||
for k, vals := range req.Header { |
||||
for _, val := range vals { |
||||
freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{ |
||||
Key: proto.String(k), |
||||
Value: proto.String(val), |
||||
}) |
||||
} |
||||
} |
||||
if methodAcceptsRequestBody[req.Method] && req.Body != nil { |
||||
// Avoid a []byte copy if req.Body has a Bytes method.
|
||||
switch b := req.Body.(type) { |
||||
case interface { |
||||
Bytes() []byte |
||||
}: |
||||
freq.Payload = b.Bytes() |
||||
default: |
||||
freq.Payload, err = ioutil.ReadAll(req.Body) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
} |
||||
|
||||
fres := &pb.URLFetchResponse{} |
||||
if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
res = &http.Response{} |
||||
res.StatusCode = int(*fres.StatusCode) |
||||
res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode)) |
||||
res.Header = make(http.Header) |
||||
res.Request = req |
||||
|
||||
// Faked:
|
||||
res.ProtoMajor = 1 |
||||
res.ProtoMinor = 1 |
||||
res.Proto = "HTTP/1.1" |
||||
res.Close = true |
||||
|
||||
for _, h := range fres.Header { |
||||
hkey := http.CanonicalHeaderKey(*h.Key) |
||||
hval := *h.Value |
||||
if hkey == "Content-Length" { |
||||
// Will get filled in below for all but HEAD requests.
|
||||
if req.Method == "HEAD" { |
||||
res.ContentLength, _ = strconv.ParseInt(hval, 10, 64) |
||||
} |
||||
continue |
||||
} |
||||
res.Header.Add(hkey, hval) |
||||
} |
||||
|
||||
if req.Method != "HEAD" { |
||||
res.ContentLength = int64(len(fres.Content)) |
||||
} |
||||
|
||||
truncated := fres.GetContentWasTruncated() |
||||
res.Body = &bodyReader{content: fres.Content, truncated: truncated} |
||||
return |
||||
} |
||||
|
||||
func init() { |
||||
internal.RegisterErrorCodeMap("urlfetch", pb.URLFetchServiceError_ErrorCode_name) |
||||
internal.RegisterTimeoutErrorCode("urlfetch", int32(pb.URLFetchServiceError_DEADLINE_EXCEEDED)) |
||||
} |
||||
Loading…
Reference in new issue