From 7717e3eb19c55d6de657d4263f064c527ae96548 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Tue, 27 Nov 2018 14:48:32 -0500 Subject: [PATCH 01/36] API-6 # Refactor the directory --- web/index.html | 48 ------------------- web/scheduler/main.go | 20 ++++++++ web/scheduler/portwardenCredentials.json | 12 +++++ web/{ => scheduler}/server/backup.go | 0 .../server/backup_controller.go | 0 web/{ => scheduler}/server/google_drive.go | 0 web/{ => scheduler}/server/server.go | 0 web/{ => scheduler}/server/token.json | 0 web/worker/main.go | 33 +++++++++++++ 9 files changed, 65 insertions(+), 48 deletions(-) delete mode 100644 web/index.html create mode 100644 web/scheduler/main.go create mode 100644 web/scheduler/portwardenCredentials.json rename web/{ => scheduler}/server/backup.go (100%) rename web/{ => scheduler}/server/backup_controller.go (100%) rename web/{ => scheduler}/server/google_drive.go (100%) rename web/{ => scheduler}/server/server.go (100%) rename web/{ => scheduler}/server/token.json (100%) create mode 100644 web/worker/main.go diff --git a/web/index.html b/web/index.html deleted file mode 100644 index c0f85a6..0000000 --- a/web/index.html +++ /dev/null @@ -1,48 +0,0 @@ - - - Melody example: chatting - - - - - -
-

Chat

-

-      
-    
- - - - \ No newline at end of file diff --git a/web/scheduler/main.go b/web/scheduler/main.go new file mode 100644 index 0000000..a63f190 --- /dev/null +++ b/web/scheduler/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "io/ioutil" + "log" + + "github.com/vwxyzjn/portwarden/web/worker/server" +) + +func main() { + credential, err := ioutil.ReadFile("portwardenCredentials.json") + if err != nil { + log.Fatalf("Unable to read client secret file: %v", err) + } + ps := server.PortwardenServer{ + Port: 5000, + GoogleDriveAppCredentials: credential, + } + ps.Run() +} diff --git a/web/scheduler/portwardenCredentials.json b/web/scheduler/portwardenCredentials.json new file mode 100644 index 0000000..6c4c448 --- /dev/null +++ b/web/scheduler/portwardenCredentials.json @@ -0,0 +1,12 @@ +{ + "web": { + "client_id": "1098948957266-vqofngan2v0282k3ia7udtue5pk9aq8n.apps.googleusercontent.com", + "project_id": "neat-tempo-223722", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://www.googleapis.com/oauth2/v3/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_secret": "N25YpAKLwoK20_XHn8zAdDO4", + "redirect_uris": ["http://localhost:5000/gdrive/login"], + "javascript_origins": ["http://localhost:5000"] + } +} diff --git a/web/server/backup.go b/web/scheduler/server/backup.go similarity index 100% rename from web/server/backup.go rename to web/scheduler/server/backup.go diff --git a/web/server/backup_controller.go b/web/scheduler/server/backup_controller.go similarity index 100% rename from web/server/backup_controller.go rename to web/scheduler/server/backup_controller.go diff --git a/web/server/google_drive.go b/web/scheduler/server/google_drive.go similarity index 100% rename from web/server/google_drive.go rename to web/scheduler/server/google_drive.go diff --git a/web/server/server.go b/web/scheduler/server/server.go similarity index 100% rename from web/server/server.go rename to web/scheduler/server/server.go diff --git a/web/server/token.json b/web/scheduler/server/token.json similarity index 100% rename from web/server/token.json rename to web/scheduler/server/token.json diff --git a/web/worker/main.go b/web/worker/main.go new file mode 100644 index 0000000..eb00413 --- /dev/null +++ b/web/worker/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "github.com/RichardKnop/machinery/v1" + "github.com/RichardKnop/machinery/v1/config" + "github.com/vwxyzjn/portwarden" +) + +func main() { + var cnf = &config.Config{ + Broker: "amqp://guest:guest@localhost:5672/", + DefaultQueue: "machinery_tasks", + ResultBackend: "amqp://guest:guest@localhost:5672/", + AMQP: &config.AMQPConfig{ + Exchange: "machinery_exchange", + ExchangeType: "direct", + BindingKey: "machinery_task", + }, + } + + server, err := machinery.NewServer(cnf) + if err != nil { + panic(err) + } + server.RegisterTasks(map[string]interface{}{ + "CreateBackupBytes": portwarden.CreateBackupBytes, + }) + worker := server.NewWorker("worker_name", 0) + err = worker.Launch() + if err != nil { + panic(err) + } +} From fa0eb81157f0d2f014ee1a2c30f011a8787ed192 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Tue, 27 Nov 2018 17:32:00 -0500 Subject: [PATCH 02/36] API-6 # Play with K8s --- web/worker/k8s.yaml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 web/worker/k8s.yaml diff --git a/web/worker/k8s.yaml b/web/worker/k8s.yaml new file mode 100644 index 0000000..7ca4db1 --- /dev/null +++ b/web/worker/k8s.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: portwarden +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello +spec: + selector: + matchLabels: + app: hello + tier: backend + track: stable + replicas: 7 + template: + metadata: + labels: + app: hello + tier: backend + track: stable + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-go-gke:1.0" + ports: + - name: http + containerPort: 80 +--- +kind: Service +apiVersion: v1 +metadata: + name: hello +spec: + selector: + app: hello + tier: backend + ports: + - protocol: TCP + port: 80 + targetPort: http \ No newline at end of file From 10d1bfe3f007de486a5a624a46e5601f99319c6c Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Tue, 27 Nov 2018 17:49:46 -0500 Subject: [PATCH 03/36] API-6 # Make sure refresh token works --- web/main.go | 2 +- web/scheduler/server/backup_controller.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/web/main.go b/web/main.go index 1a8003e..8c6a536 100644 --- a/web/main.go +++ b/web/main.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "log" - "github.com/vwxyzjn/portwarden/web/server" + "github.com/vwxyzjn/portwarden/web/scheduler/server" ) func main() { diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index 60204fb..68d1a11 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -3,8 +3,10 @@ package server import ( "net/http" + "github.com/davecgh/go-spew/spew" "github.com/gin-gonic/gin" "github.com/vwxyzjn/portwarden" + "golang.org/x/oauth2" ) const ( @@ -33,7 +35,7 @@ func EncryptBackupHandler(c *gin.Context) { // Not sure if it's supposed to call UploadFile() directly func (ps *PortwardenServer) GetGoogleDriveLoginURLHandler(c *gin.Context) { c.JSON(200, gin.H{ - "login_url": ps.GoogleDriveAppConfig.AuthCodeURL("state-token"), + "login_url": ps.GoogleDriveAppConfig.AuthCodeURL("state-token", oauth2.AccessTypeOffline, oauth2.ApprovalForce), }) return } @@ -49,7 +51,8 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) return } - GoogleDriveClient := ps.GoogleDriveAppConfig.Client(ps.GoogleDriveContext, tok) + spew.Dump(tok) + GoogleDriveClient := ps.GoogleDriveAppConfig.Client(oauth2.NoContext, tok) fileBytes := []byte("xixix") err = UploadFile(fileBytes, GoogleDriveClient, tok) if err != nil { From 5bfedbf3cc71d7fd5d72ddebdc5cd6114dcb0507 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Thu, 29 Nov 2018 10:39:10 -0500 Subject: [PATCH 04/36] API-6 # Use Google API to get user info --- web/scheduler/main.go | 2 +- web/scheduler/server/backup_controller.go | 4 +- web/scheduler/server/google_drive.go | 47 +++++++++++++++++++++++ web/scheduler/server/server.go | 2 +- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/web/scheduler/main.go b/web/scheduler/main.go index a63f190..8c6a536 100644 --- a/web/scheduler/main.go +++ b/web/scheduler/main.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "log" - "github.com/vwxyzjn/portwarden/web/worker/server" + "github.com/vwxyzjn/portwarden/web/scheduler/server" ) func main() { diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index 68d1a11..ee04e21 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -53,8 +53,8 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { } spew.Dump(tok) GoogleDriveClient := ps.GoogleDriveAppConfig.Client(oauth2.NoContext, tok) - fileBytes := []byte("xixix") - err = UploadFile(fileBytes, GoogleDriveClient, tok) + // fileBytes := []byte("xixix") + err = GetUserInfo(GoogleDriveClient, tok) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) return diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index e5481c8..7ef08c4 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -182,6 +182,53 @@ func UploadFile(fileBytes []byte, client *http.Client, token *oauth2.Token) erro return nil } +func GetUserInfo(client *http.Client, token *oauth2.Token) error { + + postURL := "https://www.googleapis.com/oauth2/v2/userinfo" + + // Extract auth or access token from Token file + // See https://godoc.org/gnolang.org/x/oauth2#Token + authToken := token.AccessToken + + // Post to Drive with RESTful method + request, err := http.NewRequest("GET", postURL, nil) + if err != nil { + return err + } + request.Header.Add("Host", "www.googleapis.com") + request.Header.Add("Authorization", "Bearer "+authToken) + request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) + + // For debugging + //fmt.Println(request) + response, err := client.Do(request) + if err != nil { + return err + } + + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return err + } + + // Output the response from Drive API + fmt.Println(string(body)) + + // // Extract the uploaded file ID to execute further customization like update the file + // jsonAPIreply, err := jason.NewObjectFromBytes(body) + // if err != nil { + // return err + // } + + // uploadedFileID, err := jsonAPIreply.GetString("id") + // if err != nil { + // return err + // } + // fmt.Println("Uploaded file ID : ", uploadedFileID) + return nil +} + /* func main() { diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 4d60230..7715561 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -25,7 +25,7 @@ type PortwardenServer struct { func (ps *PortwardenServer) Run() { var err error ps.GoogleDriveContext = context.Background() - ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, drive.DriveScope) + ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, "https://www.googleapis.com/auth/userinfo.profile", drive.DriveScope) if err != nil { log.Fatalf("Unable to parse client secret file to config: %v", err) } From 2e01ec7746f299abde5935976b9dbd73b98d1e44 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Thu, 29 Nov 2018 12:20:16 -0500 Subject: [PATCH 05/36] API-6 # play with k8s --- Gopkg.lock | 223 +++++++++++++++++++++++++++++++++----------- k8s.yaml | 151 ++++++++++++++++++++++++++++++ web/worker/k8s.yaml | 42 --------- 3 files changed, 317 insertions(+), 99 deletions(-) create mode 100644 k8s.yaml delete mode 100644 web/worker/k8s.yaml diff --git a/Gopkg.lock b/Gopkg.lock index f48ea6a..f0d1b62 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,16 +3,52 @@ [[projects]] name = "cloud.google.com/go" - packages = ["compute/metadata"] + packages = ["compute/metadata","iam","internal/optional","internal/version","pubsub","pubsub/apiv1","pubsub/internal/distribution"] revision = "74b12019e2aa53ec27882158f59192d7cd6d1998" version = "v0.33.1" +[[projects]] + branch = "master" + name = "github.com/RichardKnop/logging" + packages = ["."] + revision = "b1d5d44c82d6c9fb6c8e8dad28412163586fd8ac" + +[[projects]] + name = "github.com/RichardKnop/machinery" + packages = ["v1","v1/backends/amqp","v1/backends/dynamodb","v1/backends/eager","v1/backends/iface","v1/backends/memcache","v1/backends/mongo","v1/backends/redis","v1/backends/result","v1/brokers/amqp","v1/brokers/eager","v1/brokers/errs","v1/brokers/gcppubsub","v1/brokers/iface","v1/brokers/redis","v1/brokers/sqs","v1/common","v1/config","v1/log","v1/retry","v1/tasks","v1/tracing"] + revision = "21f6dd0def9983b08c773554c89f64b0176da3a2" + version = "v1.5.3" + +[[projects]] + name = "github.com/RichardKnop/redsync" + packages = ["."] + revision = "54b27db64c180751e7049321a2e109ff220ccdcc" + version = "v1.2.0" + [[projects]] name = "github.com/antonholmquist/jason" packages = ["."] revision = "c23cef7eaa75a6a5b8810120e167bd590d8fd2ab" version = "v1.0.0" +[[projects]] + name = "github.com/aws/aws-sdk-go" + packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/crr","aws/csm","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/ini","internal/sdkio","internal/sdkrand","internal/sdkuri","internal/shareddefaults","private/protocol","private/protocol/json/jsonutil","private/protocol/jsonrpc","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/xml/xmlutil","service/dynamodb","service/dynamodb/dynamodbattribute","service/dynamodb/dynamodbiface","service/sqs","service/sqs/sqsiface","service/sts"] + revision = "08df30d135d32f1eb21bb7754eb042aeca521a4b" + version = "v1.15.84" + +[[projects]] + name = "github.com/boj/redistore" + packages = ["."] + revision = "fc113767cd6b051980f260d6dbe84b2740c46ab0" + version = "v1.2" + +[[projects]] + branch = "master" + name = "github.com/bradfitz/gomemcache" + packages = ["memcache"] + revision = "bc664df9673713a0ccf26e3b55a673ec7301088b" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -22,16 +58,15 @@ [[projects]] branch = "master" name = "github.com/dsnet/compress" - packages = [ - ".", - "bzip2", - "bzip2/internal/sais", - "internal", - "internal/errors", - "internal/prefix" - ] + packages = [".","bzip2","bzip2/internal/sais","internal","internal/errors","internal/prefix"] revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" +[[projects]] + name = "github.com/garyburd/redigo" + packages = ["internal","redis"] + revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" + version = "v1.6.0" + [[projects]] name = "github.com/gin-contrib/cors" packages = ["."] @@ -44,20 +79,27 @@ packages = ["."] revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae" +[[projects]] + branch = "master" + name = "github.com/gin-gonic/contrib" + packages = ["sessions"] + revision = "54170a7b0b4b2f9219c79599b1b03830a5a7d68d" + [[projects]] name = "github.com/gin-gonic/gin" - packages = [ - ".", - "binding", - "json", - "render" - ] + packages = [".","binding","json","render"] revision = "b869fe1415e4b9eb52f247441830d502aece2d4d" version = "v1.3.0" +[[projects]] + branch = "master" + name = "github.com/golang/glog" + packages = ["."] + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" + [[projects]] name = "github.com/golang/protobuf" - packages = ["proto"] + packages = ["proto","protoc-gen-go/descriptor","ptypes","ptypes/any","ptypes/duration","ptypes/empty","ptypes/timestamp"] revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" @@ -67,12 +109,59 @@ packages = ["."] revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" +[[projects]] + name = "github.com/gomodule/redigo" + packages = ["internal","redis"] + revision = "9c11da706d9b7902c6da69c592f75637793fe121" + version = "v2.0.0" + +[[projects]] + name = "github.com/google/uuid" + packages = ["."] + revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" + version = "v1.1.0" + +[[projects]] + name = "github.com/googleapis/gax-go" + packages = ["."] + revision = "b001040cd31805261cbd978842099e326dfa857b" + version = "v2.0.2" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" + +[[projects]] + name = "github.com/gorilla/securecookie" + packages = ["."] + revision = "e59506cc896acb7f7bf732d4fdf5e25f7ccd8983" + version = "v1.1.1" + +[[projects]] + name = "github.com/gorilla/sessions" + packages = ["."] + revision = "f57b7e2d29c6211d16ffa52a0998272f75799030" + version = "v1.1.3" + +[[projects]] + name = "github.com/jmespath/go-jmespath" + packages = ["."] + revision = "0b12d6b5" + [[projects]] name = "github.com/json-iterator/go" packages = ["."] revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" +[[projects]] + name = "github.com/kelseyhightower/envconfig" + packages = ["."] + revision = "f611eb38b3875cc3bd991ca91c51d06446afa14c" + version = "v1.3.0" + [[projects]] name = "github.com/mattn/go-isatty" packages = ["."] @@ -103,15 +192,24 @@ revision = "cc3e4b2381762c56dbcdf9f93be03349a1dc1c14" version = "v1.0.0" +[[projects]] + name = "github.com/opentracing/opentracing-go" + packages = [".","ext","log"] + revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" + version = "v1.0.2" + [[projects]] name = "github.com/pierrec/lz4" - packages = [ - ".", - "internal/xxh32" - ] + packages = [".","internal/xxh32"] revision = "635575b42742856941dbc767b44905bb9ba083f6" version = "v2.0.7" +[[projects]] + branch = "master" + name = "github.com/streadway/amqp" + packages = ["."] + revision = "27835f1a64e97101d95306211f03c0620ffa295d" + [[projects]] branch = "master" name = "github.com/tidwall/pretty" @@ -126,15 +224,22 @@ [[projects]] name = "github.com/ulikunitz/xz" - packages = [ - ".", - "internal/hash", - "internal/xlog", - "lzma" - ] + packages = [".","internal/hash","internal/xlog","lzma"] revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" version = "v0.5.4" +[[projects]] + name = "github.com/zalando/gin-oauth2" + packages = ["google"] + revision = "8f27ddacc783deab1eb90cf225e98b6e2d7b2629" + version = "v1.5.0" + +[[projects]] + name = "go.opencensus.io" + packages = [".","exemplar","internal","internal/tagencoding","plugin/ocgrpc","plugin/ochttp","plugin/ochttp/propagation/b3","stats","stats/internal","stats/view","tag","trace","trace/internal","trace/propagation","trace/tracestate"] + revision = "b7bf3cdb64150a8c8c53b769fdeb2ba581bd4d4b" + version = "v0.18.0" + [[projects]] branch = "master" name = "golang.org/x/crypto" @@ -144,65 +249,69 @@ [[projects]] branch = "master" name = "golang.org/x/net" - packages = [ - "context", - "context/ctxhttp" - ] + packages = ["context","context/ctxhttp","http/httpguts","http2","http2/hpack","idna","internal/timeseries","trace"] revision = "adae6a3d119ae4890b46832a2e88a95adc62b8e7" [[projects]] branch = "master" name = "golang.org/x/oauth2" - packages = [ - ".", - "google", - "internal", - "jws", - "jwt" - ] + packages = [".","google","internal","jws","jwt"] revision = "f42d05182288abf10faef86d16c0d07b8d40ea2d" +[[projects]] + branch = "master" + name = "golang.org/x/sync" + packages = ["errgroup","semaphore"] + revision = "42b317875d0fa942474b76e1b46a6060d720ae6e" + [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] revision = "66b7b1311ac80bbafcd2daeef9a5e6e2cd1e2399" +[[projects]] + name = "golang.org/x/text" + packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + [[projects]] branch = "master" name = "google.golang.org/api" - packages = [ - "drive/v2", - "drive/v3", - "gensupport", - "googleapi", - "googleapi/internal/uritemplates" - ] + packages = ["drive/v2","gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","support/bundler","transport","transport/grpc","transport/http","transport/http/internal/propagation"] revision = "83a9d304b1e613fc253e1e2710778642fe81af53" [[projects]] name = "google.golang.org/appengine" - packages = [ - ".", - "internal", - "internal/app_identity", - "internal/base", - "internal/datastore", - "internal/log", - "internal/modules", - "internal/remote_api", - "internal/urlfetch", - "urlfetch" - ] + packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/socket","internal/urlfetch","socket","urlfetch"] revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" version = "v1.3.0" +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/pubsub/v1","googleapis/rpc/status","protobuf/field_mask"] + revision = "31ac5d88444a9e7ad18077db9a165d793ad06a2e" + +[[projects]] + name = "google.golang.org/grpc" + packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","credentials/oauth","encoding","encoding/proto","grpclog","internal","internal/backoff","internal/channelz","internal/envconfig","internal/grpcrand","internal/transport","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap"] + revision = "2e463a05d100327ca47ac218281906921038fd95" + version = "v1.16.0" + [[projects]] name = "gopkg.in/go-playground/validator.v8" packages = ["."] revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf" version = "v8.18.2" +[[projects]] + branch = "v2" + name = "gopkg.in/mgo.v2" + packages = [".","bson","internal/json","internal/sasl","internal/scram"] + revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5" + [[projects]] name = "gopkg.in/urfave/cli.v1" packages = ["."] @@ -218,6 +327,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "e94795c347f48a9ee90b68d79ae43762c0504d74ca19276f92b7c394faeab698" + inputs-digest = "237e8620995b0c6d6d021f63d81bfebe333496f0a6328309f3f4b3b7cf9b9d4d" solver-name = "gps-cdcl" solver-version = 1 diff --git a/k8s.yaml b/k8s.yaml new file mode 100644 index 0000000..0ad88cb --- /dev/null +++ b/k8s.yaml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: portwarden + +## Redis Definition +## https://kubernetes.io/docs/tutorials/stateless-application/guestbook/ +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis-master + labels: + app: redis +spec: + selector: + matchLabels: + app: redis + role: master + tier: backend + replicas: 1 + template: + metadata: + labels: + app: redis + role: master + tier: backend + spec: + containers: + - name: master + image: k8s.gcr.io/redis:e2e # or just image: redis + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-master + labels: + app: redis + role: master + tier: backend +spec: + ports: + - port: 6379 + targetPort: 6379 + selector: + app: redis + role: master + tier: backend +--- +apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +kind: Deployment +metadata: + name: redis-slave + labels: + app: redis +spec: + selector: + matchLabels: + app: redis + role: slave + tier: backend + replicas: 2 + template: + metadata: + labels: + app: redis + role: slave + tier: backend + spec: + containers: + - name: slave + image: gcr.io/google_samples/gb-redisslave:v1 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # Using `GET_HOSTS_FROM=dns` requires your cluster to + # provide a dns service. As of Kubernetes 1.3, DNS is a built-in + # service launched automatically. However, if the cluster you are using + # does not have a built-in DNS service, you can instead + # access an environment variable to find the master + # service's host. To do so, comment out the 'value: dns' line above, and + # uncomment the line below: + # value: env + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: redis + role: slave + tier: backend +spec: + ports: + - port: 6379 + selector: + app: redis + role: slave + tier: backend + +## Backend definition +# --- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: hello +# spec: +# selector: +# matchLabels: +# app: hello +# tier: backend +# track: stable +# replicas: 7 +# template: +# metadata: +# labels: +# app: hello +# tier: backend +# track: stable +# spec: +# containers: +# - name: hello +# image: "gcr.io/google-samples/hello-go-gke:1.0" +# ports: +# - name: http +# containerPort: 80 +# --- +# kind: Service +# apiVersion: v1 +# metadata: +# name: hello +# spec: +# selector: +# app: hello +# tier: backend +# ports: +# - protocol: TCP +# port: 80 +# targetPort: http \ No newline at end of file diff --git a/web/worker/k8s.yaml b/web/worker/k8s.yaml deleted file mode 100644 index 7ca4db1..0000000 --- a/web/worker/k8s.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: portwarden ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hello -spec: - selector: - matchLabels: - app: hello - tier: backend - track: stable - replicas: 7 - template: - metadata: - labels: - app: hello - tier: backend - track: stable - spec: - containers: - - name: hello - image: "gcr.io/google-samples/hello-go-gke:1.0" - ports: - - name: http - containerPort: 80 ---- -kind: Service -apiVersion: v1 -metadata: - name: hello -spec: - selector: - app: hello - tier: backend - ports: - - protocol: TCP - port: 80 - targetPort: http \ No newline at end of file From 45427451187568a0bc8971879806186b541e3d50 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Thu, 29 Nov 2018 14:46:01 -0500 Subject: [PATCH 06/36] API-6 # Save work --- Gopkg.lock | 52 +++------------------- core.go | 10 +++++ web/scheduler/server/backup.go | 53 +++++++++++++++++++++++ web/scheduler/server/backup_controller.go | 9 ++-- web/scheduler/server/google_drive.go | 32 ++++---------- web/scheduler/server/middleware.go | 28 ++++++++++++ web/scheduler/server/server.go | 21 +++++++++ web/worker/main.go | 6 +-- 8 files changed, 133 insertions(+), 78 deletions(-) create mode 100644 web/scheduler/server/middleware.go diff --git a/Gopkg.lock b/Gopkg.lock index f0d1b62..f55cecd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,12 +37,6 @@ revision = "08df30d135d32f1eb21bb7754eb042aeca521a4b" version = "v1.15.84" -[[projects]] - name = "github.com/boj/redistore" - packages = ["."] - revision = "fc113767cd6b051980f260d6dbe84b2740c46ab0" - version = "v1.2" - [[projects]] branch = "master" name = "github.com/bradfitz/gomemcache" @@ -61,12 +55,6 @@ packages = [".","bzip2","bzip2/internal/sais","internal","internal/errors","internal/prefix"] revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" -[[projects]] - name = "github.com/garyburd/redigo" - packages = ["internal","redis"] - revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" - version = "v1.6.0" - [[projects]] name = "github.com/gin-contrib/cors" packages = ["."] @@ -79,12 +67,6 @@ packages = ["."] revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae" -[[projects]] - branch = "master" - name = "github.com/gin-gonic/contrib" - packages = ["sessions"] - revision = "54170a7b0b4b2f9219c79599b1b03830a5a7d68d" - [[projects]] name = "github.com/gin-gonic/gin" packages = [".","binding","json","render"] @@ -92,10 +74,10 @@ version = "v1.3.0" [[projects]] - branch = "master" - name = "github.com/golang/glog" - packages = ["."] - revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" + name = "github.com/go-redis/redis" + packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto","internal/singleflight","internal/util"] + revision = "b3d9bf10f6666b2ee5100a6f3f84f4caf3b4e37d" + version = "v6.14.2" [[projects]] name = "github.com/golang/protobuf" @@ -127,24 +109,6 @@ revision = "b001040cd31805261cbd978842099e326dfa857b" version = "v2.0.2" -[[projects]] - name = "github.com/gorilla/context" - packages = ["."] - revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" - version = "v1.1.1" - -[[projects]] - name = "github.com/gorilla/securecookie" - packages = ["."] - revision = "e59506cc896acb7f7bf732d4fdf5e25f7ccd8983" - version = "v1.1.1" - -[[projects]] - name = "github.com/gorilla/sessions" - packages = ["."] - revision = "f57b7e2d29c6211d16ffa52a0998272f75799030" - version = "v1.1.3" - [[projects]] name = "github.com/jmespath/go-jmespath" packages = ["."] @@ -228,12 +192,6 @@ revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" version = "v0.5.4" -[[projects]] - name = "github.com/zalando/gin-oauth2" - packages = ["google"] - revision = "8f27ddacc783deab1eb90cf225e98b6e2d7b2629" - version = "v1.5.0" - [[projects]] name = "go.opencensus.io" packages = [".","exemplar","internal","internal/tagencoding","plugin/ocgrpc","plugin/ochttp","plugin/ochttp/propagation/b3","stats","stats/internal","stats/view","tag","trace","trace/internal","trace/propagation","trace/tracestate"] @@ -327,6 +285,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "237e8620995b0c6d6d021f63d81bfebe333496f0a6328309f3f4b3b7cf9b9d4d" + inputs-digest = "d622028d481381214ed075bf271e699c86fc8f22690364f507e3d82def387ea8" solver-name = "gps-cdcl" solver-version = 1 diff --git a/core.go b/core.go index 2a6ef3b..1d3eeb5 100644 --- a/core.go +++ b/core.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os" "os/exec" + "path/filepath" "regexp" "strconv" "strings" @@ -50,6 +51,15 @@ type LoginCredentials struct { Code string `json:"code"` } +func CreateBackupBytesUsingBitwardenLocalJSON(dataJson []byte, BITWARDENCLI_APPDATA_DIR, passphrase, sessionKey string, sleepMilliseconds int) ([]byte, error) { + // Put data.json in the BITWARDENCLI_APPDATA_DIR + defer BWLogout() + if err := ioutil.WriteFile(filepath.Join(BITWARDENCLI_APPDATA_DIR, "data.json"), dataJson, 0644); err != nil { + return nil, err + } + return CreateBackupBytes(passphrase, sessionKey, sleepMilliseconds) +} + func CreateBackupFile(fileName, passphrase, sessionKey string, sleepMilliseconds int) error { defer BWLogout() if !strings.HasSuffix(fileName, ".portwarden") { diff --git a/web/scheduler/server/backup.go b/web/scheduler/server/backup.go index 3092b65..0617496 100644 --- a/web/scheduler/server/backup.go +++ b/web/scheduler/server/backup.go @@ -1,9 +1,14 @@ package server import ( + "encoding/json" + "io/ioutil" "mime/multipart" + "net/http" + "strconv" "github.com/vwxyzjn/portwarden" + "golang.org/x/oauth2" ) const ( @@ -26,3 +31,51 @@ type GoogleDriveCredentials struct { Code string `form:"code"` Scope string `form:"scope"` } + +type PortwardenUser struct { + ID string + GoogleUserInfo GoogleUserInfo + GoogleToken *oauth2.Token + BitwardenDataJSON []byte + BitwardenSessionKey string +} + +type GoogleUserInfo struct { + ID string `json:"id"` + Name string `json:"name"` + GivenName string `json:"given_name"` + FamilyName string `json:"family_name"` + Link string `json:"link"` + Picture string `json:"picture"` + Locale string `json:"locale"` +} + +func (pu *PortwardenUser) CreateWithGoogle() error { + postURL := "https://www.googleapis.com/oauth2/v2/userinfo" + request, err := http.NewRequest("GET", postURL, nil) + if err != nil { + return err + } + request.Header.Add("Host", "www.googleapis.com") + request.Header.Add("Authorization", "Bearer "+pu.GoogleToken.AccessToken) + request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) + + // For debugging + //fmt.Println(request) + GoogleDriveClient := GoogleDriveAppConfig.Client(oauth2.NoContext, pu.GoogleToken) + response, err := GoogleDriveClient.Do(request) + if err != nil { + return err + } + + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return err + } + if err := json.Unmarshal(body, &pu.GoogleUserInfo); err != nil { + return err + } + pu.ID = pu.GoogleUserInfo.ID + return nil +} diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index ee04e21..0fd0e0d 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -51,14 +51,13 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) return } - spew.Dump(tok) - GoogleDriveClient := ps.GoogleDriveAppConfig.Client(oauth2.NoContext, tok) - // fileBytes := []byte("xixix") - err = GetUserInfo(GoogleDriveClient, tok) + // pu := &PortwardenUser{GoogleToken: tok} + body, err := GetUserInfo(tok) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ""}) return } + spew.Dump(string(body)) c.JSON(200, "Login Successful") return } diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index 7ef08c4..c843916 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -182,51 +182,37 @@ func UploadFile(fileBytes []byte, client *http.Client, token *oauth2.Token) erro return nil } -func GetUserInfo(client *http.Client, token *oauth2.Token) error { +func GetUserInfo(token *oauth2.Token) ([]byte, error) { postURL := "https://www.googleapis.com/oauth2/v2/userinfo" // Extract auth or access token from Token file // See https://godoc.org/gnolang.org/x/oauth2#Token - authToken := token.AccessToken + // authToken := token.AccessToken // Post to Drive with RESTful method request, err := http.NewRequest("GET", postURL, nil) if err != nil { - return err + return nil, err } request.Header.Add("Host", "www.googleapis.com") - request.Header.Add("Authorization", "Bearer "+authToken) + request.Header.Add("Authorization", "Bearer "+"authToken") request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) // For debugging //fmt.Println(request) - response, err := client.Do(request) + GoogleDriveClient := GoogleDriveAppConfig.Client(oauth2.NoContext, token) + response, err := GoogleDriveClient.Do(request) if err != nil { - return err + return nil, err } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { - return err + return nil, err } - - // Output the response from Drive API - fmt.Println(string(body)) - - // // Extract the uploaded file ID to execute further customization like update the file - // jsonAPIreply, err := jason.NewObjectFromBytes(body) - // if err != nil { - // return err - // } - - // uploadedFileID, err := jsonAPIreply.GetString("id") - // if err != nil { - // return err - // } - // fmt.Println("Uploaded file ID : ", uploadedFileID) - return nil + return body, nil } /* diff --git a/web/scheduler/server/middleware.go b/web/scheduler/server/middleware.go new file mode 100644 index 0000000..6bc5516 --- /dev/null +++ b/web/scheduler/server/middleware.go @@ -0,0 +1,28 @@ +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "golang.org/x/oauth2" +) + +func TokenAuthMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + code := c.Query("code") + + tok, err := GoogleDriveAppConfig.Exchange(oauth2.NoContext, code) + if err != nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) + c.Abort() + return + } + _, err = GetUserInfo(tok) + if err != nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) + c.Abort() + return + } + c.Next() + } +} diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 7715561..4edb607 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -7,12 +7,18 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "github.com/go-redis/redis" "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" drive "google.golang.org/api/drive/v2" ) +var ( + GoogleDriveAppConfig *oauth2.Config + RedisClient *redis.Client +) + type PortwardenServer struct { Port int Router *gin.Engine @@ -26,11 +32,24 @@ func (ps *PortwardenServer) Run() { var err error ps.GoogleDriveContext = context.Background() ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, "https://www.googleapis.com/auth/userinfo.profile", drive.DriveScope) + GoogleDriveAppConfig = ps.GoogleDriveAppConfig // quick hack if err != nil { log.Fatalf("Unable to parse client secret file to config: %v", err) } // ps.GoogleClient = GetClient(ps.GoogleDriveContext, ps.GoogleDriveAppConfig) + // Setup Redis + RedisClient := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", // no password set + DB: 0, // use default DB + }) + + _, err = RedisClient.Ping().Result() + if err != nil { + log.Fatalf("Unable to parse client secret file to config: %v", err) + } + ps.Router = gin.Default() ps.Router.Use(cors.Default()) @@ -41,6 +60,8 @@ func (ps *PortwardenServer) Run() { ps.Router.POST("/encrypt", EncryptBackupHandler) ps.Router.POST("/decrypt", DecryptBackupHandler) ps.Router.GET("/gdrive/loginUrl", ps.GetGoogleDriveLoginURLHandler) + + ps.Router.Use(TokenAuthMiddleware()) ps.Router.GET("/gdrive/login", ps.GetGoogleDriveLoginHandler) ps.Router.Run(":" + strconv.Itoa(ps.Port)) diff --git a/web/worker/main.go b/web/worker/main.go index eb00413..6138d92 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -8,9 +8,9 @@ import ( func main() { var cnf = &config.Config{ - Broker: "amqp://guest:guest@localhost:5672/", + Broker: "redis://localhost:6379/", DefaultQueue: "machinery_tasks", - ResultBackend: "amqp://guest:guest@localhost:5672/", + ResultBackend: "redis://localhost:6379/", AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", @@ -23,7 +23,7 @@ func main() { panic(err) } server.RegisterTasks(map[string]interface{}{ - "CreateBackupBytes": portwarden.CreateBackupBytes, + "CreateBackupBytesUsingBitwardenLocalJSON": portwarden.CreateBackupBytesUsingBitwardenLocalJSON, }) worker := server.NewWorker("worker_name", 0) err = worker.Launch() From 3ed816cbe70ac4f386e57031c580006091cfceed Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 01:01:18 -0500 Subject: [PATCH 07/36] API-6 # Get user info also gets user's email address --- web/scheduler/server/backup_controller.go | 4 ++-- web/scheduler/server/middleware.go | 8 ++++++++ web/scheduler/server/server.go | 3 +-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index 0fd0e0d..3c4f3f9 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -46,9 +46,9 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) return } - tok, err := ps.GoogleDriveAppConfig.Exchange(ps.GoogleDriveContext, gdc.Code) + tok, err := GoogleDriveAppConfig.Exchange(oauth2.NoContext, gdc.Code) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) return } // pu := &PortwardenUser{GoogleToken: tok} diff --git a/web/scheduler/server/middleware.go b/web/scheduler/server/middleware.go index 6bc5516..e64926f 100644 --- a/web/scheduler/server/middleware.go +++ b/web/scheduler/server/middleware.go @@ -1,14 +1,20 @@ package server import ( + "fmt" "net/http" "github.com/gin-gonic/gin" "golang.org/x/oauth2" ) +const ( + GoogleOauth2TokenContextVariableName = "GoogleOauth2TokenContextVariableName" +) + func TokenAuthMiddleware() gin.HandlerFunc { return func(c *gin.Context) { + fmt.Println("middleware called") code := c.Query("code") tok, err := GoogleDriveAppConfig.Exchange(oauth2.NoContext, code) @@ -23,6 +29,8 @@ func TokenAuthMiddleware() gin.HandlerFunc { c.Abort() return } + c.Set(GoogleOauth2TokenContextVariableName, tok) + fmt.Println("middleware passed") c.Next() } } diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 4edb607..2b74203 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -31,7 +31,7 @@ type PortwardenServer struct { func (ps *PortwardenServer) Run() { var err error ps.GoogleDriveContext = context.Background() - ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, "https://www.googleapis.com/auth/userinfo.profile", drive.DriveScope) + ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, "https://www.googleapis.com/auth/userinfo.profile", "email", drive.DriveScope) GoogleDriveAppConfig = ps.GoogleDriveAppConfig // quick hack if err != nil { log.Fatalf("Unable to parse client secret file to config: %v", err) @@ -61,7 +61,6 @@ func (ps *PortwardenServer) Run() { ps.Router.POST("/decrypt", DecryptBackupHandler) ps.Router.GET("/gdrive/loginUrl", ps.GetGoogleDriveLoginURLHandler) - ps.Router.Use(TokenAuthMiddleware()) ps.Router.GET("/gdrive/login", ps.GetGoogleDriveLoginHandler) ps.Router.Run(":" + strconv.Itoa(ps.Port)) From ad3cb2a185f08b3f422b38356e1354d586a89ab0 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 01:25:05 -0500 Subject: [PATCH 08/36] API-6 # rename file --- web/scheduler/server/{backup.go => model.go} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename web/scheduler/server/{backup.go => model.go} (95%) diff --git a/web/scheduler/server/backup.go b/web/scheduler/server/model.go similarity index 95% rename from web/scheduler/server/backup.go rename to web/scheduler/server/model.go index 0617496..2aa2d07 100644 --- a/web/scheduler/server/backup.go +++ b/web/scheduler/server/model.go @@ -33,7 +33,7 @@ type GoogleDriveCredentials struct { } type PortwardenUser struct { - ID string + Email string GoogleUserInfo GoogleUserInfo GoogleToken *oauth2.Token BitwardenDataJSON []byte @@ -42,6 +42,7 @@ type PortwardenUser struct { type GoogleUserInfo struct { ID string `json:"id"` + Email string `json:"email"` Name string `json:"name"` GivenName string `json:"given_name"` FamilyName string `json:"family_name"` @@ -76,6 +77,6 @@ func (pu *PortwardenUser) CreateWithGoogle() error { if err := json.Unmarshal(body, &pu.GoogleUserInfo); err != nil { return err } - pu.ID = pu.GoogleUserInfo.ID + pu.Email = pu.GoogleUserInfo.Email return nil } From 094966f20ee47c3403ae97e77bbb11677f43f840 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 01:45:35 -0500 Subject: [PATCH 09/36] API-6 # Add middleware to verify token --- web/scheduler/server/backup_controller.go | 17 +++++++----- web/scheduler/server/google_drive.go | 33 ----------------------- web/scheduler/server/middleware.go | 26 ++++++++---------- web/scheduler/server/model.go | 33 +++++++++++++++++++++++ web/scheduler/server/server.go | 7 ++++- 5 files changed, 61 insertions(+), 55 deletions(-) diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index 3c4f3f9..b93deb2 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -10,7 +10,11 @@ import ( ) const ( - ErrRetrievingOauthCode = "error retrieving oauth login credentials; try again" + ErrRetrievingOauthCode = "error retrieving oauth login credentials; try again" + ErrCreatingPortwardenUser = "error creating a portwarden user" + + FrontEndBaseAddressTest = "http://localhost:8000/" + FrontEndBaseAddressProd = "" ) func EncryptBackupHandler(c *gin.Context) { @@ -51,14 +55,15 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) return } - // pu := &PortwardenUser{GoogleToken: tok} - body, err := GetUserInfo(tok) + pu := &PortwardenUser{GoogleToken: tok} + err = pu.CreateWithGoogle() if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ""}) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) return } - spew.Dump(string(body)) - c.JSON(200, "Login Successful") + + spew.Dump(pu) + c.Redirect(http.StatusMovedPermanently, FrontEndBaseAddressTest+"?access_token="+pu.GoogleToken.AccessToken) return } diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index c843916..e5481c8 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -182,39 +182,6 @@ func UploadFile(fileBytes []byte, client *http.Client, token *oauth2.Token) erro return nil } -func GetUserInfo(token *oauth2.Token) ([]byte, error) { - - postURL := "https://www.googleapis.com/oauth2/v2/userinfo" - - // Extract auth or access token from Token file - // See https://godoc.org/gnolang.org/x/oauth2#Token - // authToken := token.AccessToken - - // Post to Drive with RESTful method - request, err := http.NewRequest("GET", postURL, nil) - if err != nil { - return nil, err - } - request.Header.Add("Host", "www.googleapis.com") - request.Header.Add("Authorization", "Bearer "+"authToken") - request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) - - // For debugging - //fmt.Println(request) - GoogleDriveClient := GoogleDriveAppConfig.Client(oauth2.NoContext, token) - response, err := GoogleDriveClient.Do(request) - if err != nil { - return nil, err - } - - defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - return body, nil -} - /* func main() { diff --git a/web/scheduler/server/middleware.go b/web/scheduler/server/middleware.go index e64926f..0a87b01 100644 --- a/web/scheduler/server/middleware.go +++ b/web/scheduler/server/middleware.go @@ -1,11 +1,11 @@ package server import ( - "fmt" "net/http" + "strconv" + "strings" "github.com/gin-gonic/gin" - "golang.org/x/oauth2" ) const ( @@ -14,23 +14,19 @@ const ( func TokenAuthMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - fmt.Println("middleware called") - code := c.Query("code") + var token string + HeaderAuthorization, ok := c.Request.Header["Authorization"] + if ok && len(HeaderAuthorization) >= 1 { + token = HeaderAuthorization[0] + token = strings.TrimPrefix(token, "Bearer ") + } - tok, err := GoogleDriveAppConfig.Exchange(oauth2.NoContext, code) - if err != nil { - c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) + verified, err := VerifyGoogleAccessToekn(token) + if err != nil || !verified { + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Verification status is " + strconv.FormatBool(verified)}) c.Abort() return } - _, err = GetUserInfo(tok) - if err != nil { - c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) - c.Abort() - return - } - c.Set(GoogleOauth2TokenContextVariableName, tok) - fmt.Println("middleware passed") c.Next() } } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index 2aa2d07..a71ae28 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "errors" "io/ioutil" "mime/multipart" "net/http" @@ -26,6 +27,17 @@ type DecryptBackupInfo struct { Passphrase string `form:"passphrase"` } +type GoogleTokenVerifyResponse struct { + IssuedTo string `json:"issued_to"` + Audience string `json:"audience"` + UserID string `json:"user_id"` + Scope string `json:"scope"` + ExpiresIn int64 `json:"expires_in"` + Email string `json:"email"` + VerifiedEmail bool `json:"verified_email"` + AccessType string `json:"access_type"` +} + type GoogleDriveCredentials struct { State string `form:"state"` Code string `form:"code"` @@ -80,3 +92,24 @@ func (pu *PortwardenUser) CreateWithGoogle() error { pu.Email = pu.GoogleUserInfo.Email return nil } + +func VerifyGoogleAccessToekn(access_token string) (bool, error) { + url := "https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=" + access_token + response, err := http.Get(url) + defer response.Body.Close() + if err != nil { + return false, err + } + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return false, err + } + var gtvr GoogleTokenVerifyResponse + if err := json.Unmarshal(body, >vr); err != nil { + return false, err + } + if !gtvr.VerifiedEmail { + return false, errors.New(string(body)) + } + return true, nil +} diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 2b74203..a4a4b78 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -57,11 +57,16 @@ func (ps *PortwardenServer) Run() { http.ServeFile(c.Writer, c.Request, "index.html") }) - ps.Router.POST("/encrypt", EncryptBackupHandler) ps.Router.POST("/decrypt", DecryptBackupHandler) ps.Router.GET("/gdrive/loginUrl", ps.GetGoogleDriveLoginURLHandler) ps.Router.GET("/gdrive/login", ps.GetGoogleDriveLoginHandler) + ps.Router.Use(TokenAuthMiddleware()) + ps.Router.GET("/test/TokenAuthMiddleware", func(c *gin.Context) { + c.JSON(200, "success") + }) + ps.Router.POST("/encrypt", EncryptBackupHandler) + ps.Router.Run(":" + strconv.Itoa(ps.Port)) } From 269452d5aa63b8afce014c4aa52e46d1430a330d Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 02:05:55 -0500 Subject: [PATCH 10/36] API-6 # Make sure the newly created redis client variable is available globally --- web/scheduler/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index a4a4b78..45ba5da 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -39,7 +39,7 @@ func (ps *PortwardenServer) Run() { // ps.GoogleClient = GetClient(ps.GoogleDriveContext, ps.GoogleDriveAppConfig) // Setup Redis - RedisClient := redis.NewClient(&redis.Options{ + RedisClient = redis.NewClient(&redis.Options{ Addr: "localhost:6379", Password: "", // no password set DB: 0, // use default DB From 78ffaf97318fc9c6af82aefb6d0d9d45cd848f8b Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 03:10:50 -0500 Subject: [PATCH 11/36] API-6 # Backup the bitwarden CLI data.json into redis --- core.go | 18 +++++++ web/scheduler/server/backup_controller.go | 32 +++++++----- web/scheduler/server/middleware.go | 1 + web/scheduler/server/model.go | 59 ++++++++++++++++++++--- web/scheduler/server/server.go | 6 +++ 5 files changed, 98 insertions(+), 18 deletions(-) diff --git a/core.go b/core.go index 1d3eeb5..3ec9294 100644 --- a/core.go +++ b/core.go @@ -205,6 +205,24 @@ func BWLoginGetSessionKey(lc *LoginCredentials) (string, error) { return sessionKey, nil } +func BWLoginGetSessionKeyAndDataJSON(lc *LoginCredentials, BITWARDENCLI_APPDATA_DIR string) (string, []byte, error) { + defer BWLogout() + sessionKey, err := BWLoginGetSessionKey(lc) + if err != nil { + return "", nil, err + } + dataJSONPath := filepath.Join(BITWARDENCLI_APPDATA_DIR, "data.json") + dat, err := ioutil.ReadFile(dataJSONPath) + if err != nil { + return "", nil, err + } + err = os.Remove(dataJSONPath) + if err != nil { + return "", nil, err + } + return sessionKey, dat, nil +} + func BWLogout() error { cmd := exec.Command("bw", "logout") return cmd.Run() diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/backup_controller.go index b93deb2..9ab9019 100644 --- a/web/scheduler/server/backup_controller.go +++ b/web/scheduler/server/backup_controller.go @@ -1,38 +1,46 @@ package server import ( + "fmt" "net/http" "github.com/davecgh/go-spew/spew" "github.com/gin-gonic/gin" - "github.com/vwxyzjn/portwarden" "golang.org/x/oauth2" ) const ( ErrRetrievingOauthCode = "error retrieving oauth login credentials; try again" ErrCreatingPortwardenUser = "error creating a portwarden user" + ErrGettingPortwardenUser = "error creating a portwarden user" + ErrLoginWithBitwarden = "error logging in with Bitwarden" FrontEndBaseAddressTest = "http://localhost:8000/" FrontEndBaseAddressProd = "" ) func EncryptBackupHandler(c *gin.Context) { - var ebi EncryptBackupInfo - if err := c.ShouldBindJSON(&ebi); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ""}) + var pu PortwardenUser + if err := c.ShouldBindJSON(&pu); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) return } - sessionKey, err := portwarden.BWLoginGetSessionKey(&ebi.BitwardenLoginCredentials) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) - return - } - err = portwarden.CreateBackupFile(ebi.FileNamePrefix, ebi.Passphrase, sessionKey, BackupDefaultSleepMilliseconds) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) + if err := pu.LoginWithBitwarden(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) return } + pu.Get() + fmt.Println(string(pu.BitwardenDataJSON)) + // sessionKey, err := portwarden.BWLoginGetSessionKey(&pu.BitwardenLoginCredentials) + // if err != nil { + // c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) + // return + // } + // err = portwarden.CreateBackupFile(pu.FileNamePrefix, pu.Passphrase, sessionKey, BackupDefaultSleepMilliseconds) + // if err != nil { + // c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) + // return + // } } //TODO: GoogleDriveHandler() will return Json with the google login url diff --git a/web/scheduler/server/middleware.go b/web/scheduler/server/middleware.go index 0a87b01..d1ca547 100644 --- a/web/scheduler/server/middleware.go +++ b/web/scheduler/server/middleware.go @@ -27,6 +27,7 @@ func TokenAuthMiddleware() gin.HandlerFunc { c.Abort() return } + c.Set(GoogleOauth2TokenContextVariableName, token) c.Next() } } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index a71ae28..ce2af1f 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -16,7 +16,7 @@ const ( BackupDefaultSleepMilliseconds = 300 ) -type EncryptBackupInfo struct { +type BackupSetting struct { FileNamePrefix string `json:"filename_prefix"` Passphrase string `json:"passphrase"` BitwardenLoginCredentials portwarden.LoginCredentials `json:"bitwarden_login_credentials"` @@ -45,11 +45,13 @@ type GoogleDriveCredentials struct { } type PortwardenUser struct { - Email string - GoogleUserInfo GoogleUserInfo - GoogleToken *oauth2.Token - BitwardenDataJSON []byte - BitwardenSessionKey string + Email string `json:"email"` + BitwardenDataJSON []byte `json:"bitwarden_data_json"` + BitwardenSessionKey string `json:"bitwarden_session_key"` + BackupSetting BackupSetting `json:"backup_setting"` + BitwardenLoginCredentials *portwarden.LoginCredentials `json:"bitwarden_login_credentials"` // Not stored in Redis + GoogleUserInfo GoogleUserInfo + GoogleToken *oauth2.Token } type GoogleUserInfo struct { @@ -90,6 +92,51 @@ func (pu *PortwardenUser) CreateWithGoogle() error { return err } pu.Email = pu.GoogleUserInfo.Email + err = pu.Set() + if err != nil { + return err + } + return nil +} + +func (pu *PortwardenUser) LoginWithBitwarden() error { + opu := PortwardenUser{Email: pu.Email} + err := opu.Get() + if err != nil { + return err + } + opu.BitwardenSessionKey, opu.BitwardenDataJSON, err = portwarden.BWLoginGetSessionKeyAndDataJSON(pu.BitwardenLoginCredentials, BITWARDENCLI_APPDATA_DIR) + if err != nil { + return err + } + err = opu.Set() + if err != nil { + return err + } + return nil +} + +func (pu *PortwardenUser) Set() error { + pu.BitwardenLoginCredentials = &portwarden.LoginCredentials{} + puJson, err := json.Marshal(pu) + if err != nil { + return err + } + err = RedisClient.Set(pu.Email, string(puJson), 0).Err() + if err != nil { + panic(err) + } + return nil +} + +func (pu *PortwardenUser) Get() error { + val, err := RedisClient.Get(pu.Email).Result() + if err != nil { + return err + } + if err := json.Unmarshal([]byte(val), &pu); err != nil { + return err + } return nil } diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 45ba5da..fc76169 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -3,6 +3,7 @@ package server import ( "log" "net/http" + "os" "strconv" "github.com/gin-contrib/cors" @@ -17,6 +18,8 @@ import ( var ( GoogleDriveAppConfig *oauth2.Config RedisClient *redis.Client + + BITWARDENCLI_APPDATA_DIR string ) type PortwardenServer struct { @@ -50,6 +53,9 @@ func (ps *PortwardenServer) Run() { log.Fatalf("Unable to parse client secret file to config: %v", err) } + // Get Bitwarden CLI Env Var + BITWARDENCLI_APPDATA_DIR = os.Getenv("BITWARDENCLI_APPDATA_DIR") + ps.Router = gin.Default() ps.Router.Use(cors.Default()) From e2a4d3fcf0924d58948ab357b4b5829cea5f0e0b Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 20:15:08 -0500 Subject: [PATCH 12/36] API-6 # rename file --- web/scheduler/server/{backup_controller.go => controller.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename web/scheduler/server/{backup_controller.go => controller.go} (100%) diff --git a/web/scheduler/server/backup_controller.go b/web/scheduler/server/controller.go similarity index 100% rename from web/scheduler/server/backup_controller.go rename to web/scheduler/server/controller.go From 7f5c319e46cff93dbdde1b77a3a95a3db1c4340e Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 20:16:08 -0500 Subject: [PATCH 13/36] API-6 # Introduce docker to the porject --- .dockerignore | 2 ++ Dockerfile | 18 ++++++++++++++++++ build.sh | 7 +++++++ docker-compose.yml | 28 ++++++++++++++++++++++++++++ web/scheduler/server/server.go | 2 +- 5 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 build.sh create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..afbc551 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +vendor +Dockerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3a189f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.11.2-alpine3.8 + +# Setup work directory +COPY . /go/src/github.com/vwxyzjn/portwarden +WORKDIR /go/src/github.com/vwxyzjn/portwarden/web/scheduler + +# Install Go Dep +RUN wget -q https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 +RUN mv dep-linux-amd64 /usr/bin/dep +RUN chmod +x /usr/bin/dep + +# Run dep +# Notice git is the dependency for running dep +RUN apk add --no-cache bash git openssh +RUN cd /go/src/github.com/vwxyzjn/portwarden && dep ensure --vendor-only + +# Ready to run +EXPOSE 5000 diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..b72bb80 --- /dev/null +++ b/build.sh @@ -0,0 +1,7 @@ +## The folloiwng commands should be executed separately. + +docker build -t vwxyzjn/portwarden-scheduler:0.0.1 . + +# minikube docker-env + +docker-compose up \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c4a55bc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3' + +services: + + scheduler: + image: vwxyzjn/portwarden-scheduler:0.0.1 + stdin_open: true + tty: true + depends_on: + - redis + ports: + - 5000:5000 + volumes: + - .:/go/src/github.com/vwxyzjn/portwarden + + redis: + image: redis + ports: + - 6379:6379 + + worker: + image: vwxyzjn/portwarden-scheduler:0.0.1 + stdin_open: true + tty: true + depends_on: + - redis + volumes: + - .:/go/src/github.com/vwxyzjn/portwarden diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index fc76169..11b3aaf 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -43,7 +43,7 @@ func (ps *PortwardenServer) Run() { // Setup Redis RedisClient = redis.NewClient(&redis.Options{ - Addr: "localhost:6379", + Addr: "redis:6379", Password: "", // no password set DB: 0, // use default DB }) From aecfc041ba10912627a1896ed5c5dc5fbcfb2de1 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 20:19:48 -0500 Subject: [PATCH 14/36] API-6 # Delete unnecessary file --- web/main.go | 20 -------------------- web/worker/main.go | 4 ++-- 2 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 web/main.go diff --git a/web/main.go b/web/main.go deleted file mode 100644 index 8c6a536..0000000 --- a/web/main.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - - "github.com/vwxyzjn/portwarden/web/scheduler/server" -) - -func main() { - credential, err := ioutil.ReadFile("portwardenCredentials.json") - if err != nil { - log.Fatalf("Unable to read client secret file: %v", err) - } - ps := server.PortwardenServer{ - Port: 5000, - GoogleDriveAppCredentials: credential, - } - ps.Run() -} diff --git a/web/worker/main.go b/web/worker/main.go index 6138d92..9629524 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -8,9 +8,9 @@ import ( func main() { var cnf = &config.Config{ - Broker: "redis://localhost:6379/", + Broker: "redis://redis:6379/", DefaultQueue: "machinery_tasks", - ResultBackend: "redis://localhost:6379/", + ResultBackend: "redis://redis:6379/", AMQP: &config.AMQPConfig{ Exchange: "machinery_exchange", ExchangeType: "direct", From df18cff9b8589ce23f4f73899d4e0f9b7cf536af Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sat, 1 Dec 2018 20:20:27 -0500 Subject: [PATCH 15/36] API-6 # Delete unnecessary files --- web/credentials.json | 1 - web/portwardenCredentials.json | 12 ------------ 2 files changed, 13 deletions(-) delete mode 100644 web/credentials.json delete mode 100644 web/portwardenCredentials.json diff --git a/web/credentials.json b/web/credentials.json deleted file mode 100644 index da7263b..0000000 --- a/web/credentials.json +++ /dev/null @@ -1 +0,0 @@ -{"installed":{"client_id":"266239424585-t42ok0rq3f9t3plllutq5vrcg7e5p9i5.apps.googleusercontent.com","project_id":"portwarden","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://www.googleapis.com/oauth2/v3/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"uV2tgocN2ha98uwC5Iv5uNiT","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file diff --git a/web/portwardenCredentials.json b/web/portwardenCredentials.json deleted file mode 100644 index 6c4c448..0000000 --- a/web/portwardenCredentials.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "web": { - "client_id": "1098948957266-vqofngan2v0282k3ia7udtue5pk9aq8n.apps.googleusercontent.com", - "project_id": "neat-tempo-223722", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://www.googleapis.com/oauth2/v3/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "N25YpAKLwoK20_XHn8zAdDO4", - "redirect_uris": ["http://localhost:5000/gdrive/login"], - "javascript_origins": ["http://localhost:5000"] - } -} From c2ae08e8a0f8ec9b0e7fcbe834d2f5731c061c79 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 00:49:52 -0500 Subject: [PATCH 16/36] API-6 # Make everything work. Now the worker backup the vault into the portwarden folder of user's google drive --- Dockerfile | 18 +- docker-compose.yml | 16 +- k8s.yaml | 188 ++++++------------ web/common.go | 71 +++++++ .../portwardenCredentials.json | 0 web/scheduler/main.go | 10 +- web/scheduler/server/controller.go | 30 ++- web/scheduler/server/google_drive.go | 102 +++++----- web/scheduler/server/model.go | 36 +++- web/scheduler/server/server.go | 37 ---- web/worker/main.go | 48 +++-- 11 files changed, 280 insertions(+), 276 deletions(-) create mode 100644 web/common.go rename web/{scheduler => }/portwardenCredentials.json (100%) diff --git a/Dockerfile b/Dockerfile index 3a189f9..dcc893c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,14 @@ -FROM golang:1.11.2-alpine3.8 +FROM ubuntu:latest + +# Install Go +RUN apt-get update +RUN apt-get install -y wget git gcc unzip +RUN wget -q -P /tmp https://dl.google.com/go/go1.11.2.linux-amd64.tar.gz +RUN tar -C /usr/local -xzf /tmp/go1.11.2.linux-amd64.tar.gz +RUN rm /tmp/go1.11.2.linux-amd64.tar.gz +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" # Setup work directory COPY . /go/src/github.com/vwxyzjn/portwarden @@ -9,9 +19,13 @@ RUN wget -q https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd RUN mv dep-linux-amd64 /usr/bin/dep RUN chmod +x /usr/bin/dep +# Install Bitwarden CLI +RUN wget -q https://github.com/bitwarden/cli/releases/download/v1.6.0/bw-linux-1.6.0.zip +RUN unzip bw-linux-1.6.0.zip -d /usr/bin/ +RUN chmod +x /usr/bin/bw + # Run dep # Notice git is the dependency for running dep -RUN apk add --no-cache bash git openssh RUN cd /go/src/github.com/vwxyzjn/portwarden && dep ensure --vendor-only # Ready to run diff --git a/docker-compose.yml b/docker-compose.yml index c4a55bc..6521f45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,9 +3,11 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-scheduler:0.0.1 + image: vwxyzjn/portwarden-scheduler:0.1.1 stdin_open: true tty: true + environment: + - BITWARDENCLI_APPDATA_DIR=/BitwardenCLI depends_on: - redis ports: @@ -19,10 +21,20 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-scheduler:0.0.1 + image: vwxyzjn/portwarden-scheduler:0.1.1 stdin_open: true tty: true + environment: + - BITWARDENCLI_APPDATA_DIR=/BitwardenCLI depends_on: - redis volumes: - .:/go/src/github.com/vwxyzjn/portwarden + + redis-commander: + image: rediscommander/redis-commander:latest + restart: always + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8081:8081" diff --git a/k8s.yaml b/k8s.yaml index 0ad88cb..5012440 100644 --- a/k8s.yaml +++ b/k8s.yaml @@ -3,149 +3,91 @@ kind: Namespace metadata: name: portwarden -## Redis Definition -## https://kubernetes.io/docs/tutorials/stateless-application/guestbook/ + --- -apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: - name: redis-master + namespace: portwarden + name: portwarden-scheduler labels: - app: redis + app: portwarden-scheduler +spec: + selector: + matchLabels: + app: portwarden-scheduler + tier: backend + # replicas: 7 + template: + metadata: + labels: + app: portwarden-scheduler + tier: backend + spec: + containers: + - name: portwarden-scheduler + image: vwxyzjn/portwarden-scheduler:0.0.1 + ports: + - name: http + containerPort: 5000 + stdin: true + tty: true +# # START of dev related lines, local mount doesn't work on windows +# volumeMounts: +# - mountPath: /go/src/github.com/vwxyzjn/portwarden +# name: dev-volume +# volumes: +# - name: dev-volume +# hostPath: +# path: /data +# # END of dev related lines +--- +kind: Service +apiVersion: v1 +metadata: + namespace: portwarden + name: portwarden-scheduler +spec: + selector: + app: portwarden-scheduler + tier: backend + ports: + - port: 5000 + targetPort: 5000 + + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: portwarden + name: redis spec: selector: matchLabels: app: redis - role: master tier: backend - replicas: 1 template: metadata: labels: app: redis - role: master tier: backend spec: containers: - - name: master - image: k8s.gcr.io/redis:e2e # or just image: redis - resources: - requests: - cpu: 100m - memory: 100Mi - ports: - - containerPort: 6379 + - name: redis + image: redis + ports: + - containerPort: 6379 --- -apiVersion: v1 kind: Service +apiVersion: v1 metadata: - name: redis-master - labels: - app: redis - role: master - tier: backend + namespace: portwarden + name: redis spec: + selector: + app: redis + tier: backend ports: - port: 6379 targetPort: 6379 - selector: - app: redis - role: master - tier: backend ---- -apiVersion: apps/v1 # for versions before 1.9.0 use apps/v1beta2 -kind: Deployment -metadata: - name: redis-slave - labels: - app: redis -spec: - selector: - matchLabels: - app: redis - role: slave - tier: backend - replicas: 2 - template: - metadata: - labels: - app: redis - role: slave - tier: backend - spec: - containers: - - name: slave - image: gcr.io/google_samples/gb-redisslave:v1 - resources: - requests: - cpu: 100m - memory: 100Mi - env: - - name: GET_HOSTS_FROM - value: dns - # Using `GET_HOSTS_FROM=dns` requires your cluster to - # provide a dns service. As of Kubernetes 1.3, DNS is a built-in - # service launched automatically. However, if the cluster you are using - # does not have a built-in DNS service, you can instead - # access an environment variable to find the master - # service's host. To do so, comment out the 'value: dns' line above, and - # uncomment the line below: - # value: env - ports: - - containerPort: 6379 ---- -apiVersion: v1 -kind: Service -metadata: - name: redis-slave - labels: - app: redis - role: slave - tier: backend -spec: - ports: - - port: 6379 - selector: - app: redis - role: slave - tier: backend - -## Backend definition -# --- -# apiVersion: apps/v1 -# kind: Deployment -# metadata: -# name: hello -# spec: -# selector: -# matchLabels: -# app: hello -# tier: backend -# track: stable -# replicas: 7 -# template: -# metadata: -# labels: -# app: hello -# tier: backend -# track: stable -# spec: -# containers: -# - name: hello -# image: "gcr.io/google-samples/hello-go-gke:1.0" -# ports: -# - name: http -# containerPort: 80 -# --- -# kind: Service -# apiVersion: v1 -# metadata: -# name: hello -# spec: -# selector: -# app: hello -# tier: backend -# ports: -# - protocol: TCP -# port: 80 -# targetPort: http \ No newline at end of file diff --git a/web/common.go b/web/common.go new file mode 100644 index 0000000..7b9b8a1 --- /dev/null +++ b/web/common.go @@ -0,0 +1,71 @@ +package web + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + + machinery "github.com/RichardKnop/machinery/v1" + "github.com/RichardKnop/machinery/v1/config" + "github.com/go-redis/redis" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + drive "google.golang.org/api/drive/v2" +) + +const ( + BackupDefaultSleepMilliseconds = 300 +) + +var ( + GoogleDriveAppConfig *oauth2.Config + RedisClient *redis.Client + MachineryServer *machinery.Server + BITWARDENCLI_APPDATA_DIR string +) + +func InitCommonVars() { + var err error + + // Setup Redis + RedisClient = redis.NewClient(&redis.Options{ + Addr: "redis:6379", + Password: "", // no password set + DB: 0, // use default DB + }) + _, err = RedisClient.Ping().Result() + if err != nil { + log.Fatalf("Unable to parse client secret file to config: %v", err) + } + + // Setup Machinery + var cnf = &config.Config{ + Broker: "redis://redis:6379/", + DefaultQueue: "machinery_tasks", + ResultBackend: "redis://redis:6379/", + AMQP: &config.AMQPConfig{ + Exchange: "machinery_exchange", + ExchangeType: "direct", + BindingKey: "machinery_task", + }, + } + MachineryServer, err = machinery.NewServer(cnf) + if err != nil { + panic(err) + } + + // Setup Google things + absPath, err := filepath.Abs("../portwardenCredentials.json") + if err != nil { + panic(err) + } + credential, err := ioutil.ReadFile(absPath) + if err != nil { + log.Fatalf("Unable to read client secret file: %v", err) + } + GoogleDriveAppConfig, err = google.ConfigFromJSON(credential, "https://www.googleapis.com/auth/userinfo.profile", "email", drive.DriveScope) + + // Get Bitwarden CLI Env Var + BITWARDENCLI_APPDATA_DIR = os.Getenv("BITWARDENCLI_APPDATA_DIR") +} diff --git a/web/scheduler/portwardenCredentials.json b/web/portwardenCredentials.json similarity index 100% rename from web/scheduler/portwardenCredentials.json rename to web/portwardenCredentials.json diff --git a/web/scheduler/main.go b/web/scheduler/main.go index 8c6a536..c0ae818 100644 --- a/web/scheduler/main.go +++ b/web/scheduler/main.go @@ -1,20 +1,14 @@ package main import ( - "io/ioutil" - "log" - + "github.com/vwxyzjn/portwarden/web" "github.com/vwxyzjn/portwarden/web/scheduler/server" ) func main() { - credential, err := ioutil.ReadFile("portwardenCredentials.json") - if err != nil { - log.Fatalf("Unable to read client secret file: %v", err) - } + web.InitCommonVars() ps := server.PortwardenServer{ Port: 5000, - GoogleDriveAppCredentials: credential, } ps.Run() } diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index 9ab9019..7c290b7 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -1,19 +1,21 @@ package server import ( - "fmt" "net/http" "github.com/davecgh/go-spew/spew" "github.com/gin-gonic/gin" + "github.com/vwxyzjn/portwarden/web" "golang.org/x/oauth2" ) const ( + ErrBindingFromGin = "(debugging message) error binding json" ErrRetrievingOauthCode = "error retrieving oauth login credentials; try again" ErrCreatingPortwardenUser = "error creating a portwarden user" - ErrGettingPortwardenUser = "error creating a portwarden user" + ErrGettingPortwardenUser = "error getting a portwarden user" ErrLoginWithBitwarden = "error logging in with Bitwarden" + ErrSettingupBackup = "error setting up backup" FrontEndBaseAddressTest = "http://localhost:8000/" FrontEndBaseAddressProd = "" @@ -22,25 +24,21 @@ const ( func EncryptBackupHandler(c *gin.Context) { var pu PortwardenUser if err := c.ShouldBindJSON(&pu); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrBindingFromGin}) return } if err := pu.LoginWithBitwarden(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) return } - pu.Get() - fmt.Println(string(pu.BitwardenDataJSON)) - // sessionKey, err := portwarden.BWLoginGetSessionKey(&pu.BitwardenLoginCredentials) - // if err != nil { - // c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) - // return - // } - // err = portwarden.CreateBackupFile(pu.FileNamePrefix, pu.Passphrase, sessionKey, BackupDefaultSleepMilliseconds) - // if err != nil { - // c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": sessionKey}) - // return - // } + if err := pu.Get(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) + return + } + if err := pu.SetupAutomaticBackup(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrSettingupBackup}) + return + } } //TODO: GoogleDriveHandler() will return Json with the google login url @@ -58,7 +56,7 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrRetrievingOauthCode}) return } - tok, err := GoogleDriveAppConfig.Exchange(oauth2.NoContext, gdc.Code) + tok, err := web.GoogleDriveAppConfig.Exchange(oauth2.NoContext, gdc.Code) if err != nil { c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) return diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index e5481c8..09926b6 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -1,22 +1,23 @@ package server import ( + "bytes" "crypto/rand" "encoding/json" "fmt" - "io/ioutil" "log" "net/http" "net/url" "os" "os/user" "path/filepath" - "strconv" - "strings" + "time" + + "github.com/vwxyzjn/portwarden/web" - "github.com/antonholmquist/jason" "golang.org/x/net/context" "golang.org/x/oauth2" + drive "google.golang.org/api/drive/v2" ) // GetClient uses a Context and Config to retrieve a Token @@ -115,73 +116,62 @@ func randStr(strSize int, randType string) string { return string(bytes) } -// Multipart upload method -// See https://developers.google.com/drive/v3/web/manage-uploads -// uploadFile() takes in the encrypted byte array to be uploaded, client generated by GetClient() and oauth2 token -func UploadFile(fileBytes []byte, client *http.Client, token *oauth2.Token) error { - - fileMIMEType := http.DetectContentType(fileBytes) - - postURL := "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart" - - // Extract auth or access token from Token file - // See https://godoc.org/gnolang.org/x/oauth2#Token - authToken := token.AccessToken - - boundary := randStr(32, "alphanum") - - uploadData := []byte("\n" + - "--" + boundary + "\n" + - "Content-Type: application/json; charset=" + string('"') + "UTF-8" + string('"') + "\n\n" + - "{ \n" + - string('"') + "name" + string('"') + ":" + string('"') + "test" + string('"') + "\n" + - "} \n\n" + - "--" + boundary + "\n" + - "Content-Type:" + fileMIMEType + "\n\n" + - string(fileBytes) + "\n" + - - "--" + boundary + "--") - - // Post to Drive with RESTful method - request, err := http.NewRequest("POST", postURL, strings.NewReader(string(uploadData))) +// UploadFile upload the fileBytes to Google Drive's portwarden folder +// https://gist.github.com/tzmartin/f5732091783752660b671c20479f519a +func UploadFile(fileBytes []byte, token *oauth2.Token) error { + client := web.GoogleDriveAppConfig.Client(oauth2.NoContext, token) + srv, err := drive.New(client) if err != nil { return err } - request.Header.Add("Host", "www.googleapis.com") - request.Header.Add("Authorization", "Bearer "+authToken) - request.Header.Add("Content-Type", "multipart/related; boundary="+string('"')+boundary+string('"')) - request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) + mimeType := http.DetectContentType(fileBytes) - // For debugging - //fmt.Println(request) - response, err := client.Do(request) + parentId, err := GetOrCreateFolder(srv, "portwarden_backup") if err != nil { return err } - defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) + fmt.Println("Start upload") + f := &drive.File{Title: time.Now().Format("01-02-2006") + ".portwarden", MimeType: mimeType} + if parentId != "" { + p := &drive.ParentReference{Id: parentId} + f.Parents = []*drive.ParentReference{p} + } + + _, err = srv.Files.Insert(f).Media(bytes.NewReader(fileBytes)).Do() if err != nil { return err } - // Output the response from Drive API - fmt.Println(string(body)) - - // Extract the uploaded file ID to execute further customization like update the file - jsonAPIreply, err := jason.NewObjectFromBytes(body) - if err != nil { - return err - } - - uploadedFileID, err := jsonAPIreply.GetString("id") - if err != nil { - return err - } - fmt.Println("Uploaded file ID : ", uploadedFileID) return nil } +func GetOrCreateFolder(srv *drive.Service, folderName string) (string, error) { + folderId := "" + if folderName == "" { + return "", nil + } + q := fmt.Sprintf("title=\"%s\" and mimeType=\"application/vnd.google-apps.folder\"", folderName) + + r, err := srv.Files.List().Q(q).MaxResults(1).Do() + if err != nil { + return "", err + } + + if len(r.Items) > 0 { + folderId = r.Items[0].Id + } else { + // no folder found create new + f := &drive.File{Title: folderName, Description: "Auto Create by gdrive-upload", MimeType: "application/vnd.google-apps.folder"} + r, err := srv.Files.Insert(f).Do() + if err != nil { + return "", err + } + folderId = r.Id + } + return folderId, nil +} + /* func main() { diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index ce2af1f..4489ece 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -8,18 +8,15 @@ import ( "net/http" "strconv" + "github.com/RichardKnop/machinery/v1/tasks" "github.com/vwxyzjn/portwarden" + "github.com/vwxyzjn/portwarden/web" "golang.org/x/oauth2" ) -const ( - BackupDefaultSleepMilliseconds = 300 -) - type BackupSetting struct { - FileNamePrefix string `json:"filename_prefix"` - Passphrase string `json:"passphrase"` - BitwardenLoginCredentials portwarden.LoginCredentials `json:"bitwarden_login_credentials"` + FileNamePrefix string `json:"filename_prefix"` + Passphrase string `json:"passphrase"` } type DecryptBackupInfo struct { @@ -77,7 +74,7 @@ func (pu *PortwardenUser) CreateWithGoogle() error { // For debugging //fmt.Println(request) - GoogleDriveClient := GoogleDriveAppConfig.Client(oauth2.NoContext, pu.GoogleToken) + GoogleDriveClient := web.GoogleDriveAppConfig.Client(oauth2.NoContext, pu.GoogleToken) response, err := GoogleDriveClient.Do(request) if err != nil { return err @@ -105,7 +102,7 @@ func (pu *PortwardenUser) LoginWithBitwarden() error { if err != nil { return err } - opu.BitwardenSessionKey, opu.BitwardenDataJSON, err = portwarden.BWLoginGetSessionKeyAndDataJSON(pu.BitwardenLoginCredentials, BITWARDENCLI_APPDATA_DIR) + opu.BitwardenSessionKey, opu.BitwardenDataJSON, err = portwarden.BWLoginGetSessionKeyAndDataJSON(pu.BitwardenLoginCredentials, web.BITWARDENCLI_APPDATA_DIR) if err != nil { return err } @@ -116,13 +113,30 @@ func (pu *PortwardenUser) LoginWithBitwarden() error { return nil } +func (pu *PortwardenUser) SetupAutomaticBackup() error { + signature := &tasks.Signature{ + Name: "BackupToGoogleDrive", + Args: []tasks.Arg{ + { + Type: "string", + Value: pu.Email, + }, + }, + } + _, err := web.MachineryServer.SendTask(signature) + if err != nil { + return err + } + return nil +} + func (pu *PortwardenUser) Set() error { pu.BitwardenLoginCredentials = &portwarden.LoginCredentials{} puJson, err := json.Marshal(pu) if err != nil { return err } - err = RedisClient.Set(pu.Email, string(puJson), 0).Err() + err = web.RedisClient.Set(pu.Email, string(puJson), 0).Err() if err != nil { panic(err) } @@ -130,7 +144,7 @@ func (pu *PortwardenUser) Set() error { } func (pu *PortwardenUser) Get() error { - val, err := RedisClient.Get(pu.Email).Result() + val, err := web.RedisClient.Get(pu.Email).Result() if err != nil { return err } diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 11b3aaf..735c280 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -1,25 +1,13 @@ package server import ( - "log" "net/http" - "os" "strconv" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" - "github.com/go-redis/redis" "golang.org/x/net/context" "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - drive "google.golang.org/api/drive/v2" -) - -var ( - GoogleDriveAppConfig *oauth2.Config - RedisClient *redis.Client - - BITWARDENCLI_APPDATA_DIR string ) type PortwardenServer struct { @@ -28,34 +16,9 @@ type PortwardenServer struct { GoogleDriveContext context.Context GoogleDriveAppCredentials []byte GoogleDriveAppConfig *oauth2.Config - GoogleClient *http.Client } func (ps *PortwardenServer) Run() { - var err error - ps.GoogleDriveContext = context.Background() - ps.GoogleDriveAppConfig, err = google.ConfigFromJSON(ps.GoogleDriveAppCredentials, "https://www.googleapis.com/auth/userinfo.profile", "email", drive.DriveScope) - GoogleDriveAppConfig = ps.GoogleDriveAppConfig // quick hack - if err != nil { - log.Fatalf("Unable to parse client secret file to config: %v", err) - } - // ps.GoogleClient = GetClient(ps.GoogleDriveContext, ps.GoogleDriveAppConfig) - - // Setup Redis - RedisClient = redis.NewClient(&redis.Options{ - Addr: "redis:6379", - Password: "", // no password set - DB: 0, // use default DB - }) - - _, err = RedisClient.Ping().Result() - if err != nil { - log.Fatalf("Unable to parse client secret file to config: %v", err) - } - - // Get Bitwarden CLI Env Var - BITWARDENCLI_APPDATA_DIR = os.Getenv("BITWARDENCLI_APPDATA_DIR") - ps.Router = gin.Default() ps.Router.Use(cors.Default()) diff --git a/web/worker/main.go b/web/worker/main.go index 9629524..63febc1 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -1,33 +1,39 @@ package main import ( - "github.com/RichardKnop/machinery/v1" - "github.com/RichardKnop/machinery/v1/config" + "fmt" + "github.com/vwxyzjn/portwarden" + "github.com/vwxyzjn/portwarden/web" + "github.com/vwxyzjn/portwarden/web/scheduler/server" ) func main() { - var cnf = &config.Config{ - Broker: "redis://redis:6379/", - DefaultQueue: "machinery_tasks", - ResultBackend: "redis://redis:6379/", - AMQP: &config.AMQPConfig{ - Exchange: "machinery_exchange", - ExchangeType: "direct", - BindingKey: "machinery_task", - }, - } - - server, err := machinery.NewServer(cnf) - if err != nil { - panic(err) - } - server.RegisterTasks(map[string]interface{}{ - "CreateBackupBytesUsingBitwardenLocalJSON": portwarden.CreateBackupBytesUsingBitwardenLocalJSON, + web.InitCommonVars() + web.MachineryServer.RegisterTasks(map[string]interface{}{ + "BackupToGoogleDrive": BackupToGoogleDrive, }) - worker := server.NewWorker("worker_name", 0) - err = worker.Launch() + worker := web.MachineryServer.NewWorker("worker_name", 0) + err := worker.Launch() if err != nil { panic(err) } } + +func BackupToGoogleDrive(email string) error { + pu := server.PortwardenUser{Email: email} + fmt.Println(pu.Email) + err := pu.Get() + if err != nil { + return err + } + encryptedData, err := portwarden.CreateBackupBytesUsingBitwardenLocalJSON(pu.BitwardenDataJSON, web.BITWARDENCLI_APPDATA_DIR, pu.BackupSetting.Passphrase, pu.BitwardenSessionKey, web.BackupDefaultSleepMilliseconds) + if err != nil { + return err + } + err = server.UploadFile(encryptedData, pu.GoogleToken) + if err != nil { + return err + } + return nil +} From f68909400714056f0d6259697f0981893f3cde98 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 00:51:31 -0500 Subject: [PATCH 17/36] API-6 # Refactor --- web/common.go | 3 ++- web/scheduler/server/google_drive.go | 28 +--------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/web/common.go b/web/common.go index 7b9b8a1..39fd24b 100644 --- a/web/common.go +++ b/web/common.go @@ -15,7 +15,8 @@ import ( ) const ( - BackupDefaultSleepMilliseconds = 300 + BackupDefaultSleepMilliseconds = 300 + PortwardenGoogleDriveBackupFolderName = "portwarden_backup" ) var ( diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index 09926b6..8c12de9 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -2,7 +2,6 @@ package server import ( "bytes" - "crypto/rand" "encoding/json" "fmt" "log" @@ -92,30 +91,6 @@ func SaveToken(file string, token *oauth2.Token) { json.NewEncoder(f).Encode(token) } -func randStr(strSize int, randType string) string { - - var dictionary string - - if randType == "alphanum" { - dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - } - - if randType == "alpha" { - dictionary = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - } - - if randType == "number" { - dictionary = "0123456789" - } - - var bytes = make([]byte, strSize) - rand.Read(bytes) - for k, v := range bytes { - bytes[k] = dictionary[v%byte(len(dictionary))] - } - return string(bytes) -} - // UploadFile upload the fileBytes to Google Drive's portwarden folder // https://gist.github.com/tzmartin/f5732091783752660b671c20479f519a func UploadFile(fileBytes []byte, token *oauth2.Token) error { @@ -126,12 +101,11 @@ func UploadFile(fileBytes []byte, token *oauth2.Token) error { } mimeType := http.DetectContentType(fileBytes) - parentId, err := GetOrCreateFolder(srv, "portwarden_backup") + parentId, err := GetOrCreateFolder(srv, web.PortwardenGoogleDriveBackupFolderName) if err != nil { return err } - fmt.Println("Start upload") f := &drive.File{Title: time.Now().Format("01-02-2006") + ".portwarden", MimeType: mimeType} if parentId != "" { p := &drive.ParentReference{Id: parentId} From 947e31633239be85757e7dd87da87cfb70084590 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 01:04:21 -0500 Subject: [PATCH 18/36] API-6 # Make sure the passphrase is passed to the Portwarden User --- web/scheduler/server/controller.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index 7c290b7..d7186bb 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -23,6 +23,7 @@ const ( func EncryptBackupHandler(c *gin.Context) { var pu PortwardenUser + var opu PortwardenUser if err := c.ShouldBindJSON(&pu); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrBindingFromGin}) return @@ -31,14 +32,20 @@ func EncryptBackupHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) return } - if err := pu.Get(); err != nil { + opu.Email = pu.Email + if err := opu.Get(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) return } - if err := pu.SetupAutomaticBackup(); err != nil { + opu.BackupSetting = pu.BackupSetting + if err := opu.SetupAutomaticBackup(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrSettingupBackup}) return } + if err := opu.Set(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) + return + } } //TODO: GoogleDriveHandler() will return Json with the google login url From 60be4664e07877974389628b5729521396dd76ec Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 01:17:48 -0500 Subject: [PATCH 19/36] API-6 # Automatic backup is set --- web/common.go | 1 + web/scheduler/server/controller.go | 2 +- web/scheduler/server/model.go | 9 ++++++--- web/worker/main.go | 3 +++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/web/common.go b/web/common.go index 39fd24b..0a0350c 100644 --- a/web/common.go +++ b/web/common.go @@ -17,6 +17,7 @@ import ( const ( BackupDefaultSleepMilliseconds = 300 PortwardenGoogleDriveBackupFolderName = "portwarden_backup" + MachineryRetryCount = 3 ) var ( diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index d7186bb..ac8d775 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -38,7 +38,7 @@ func EncryptBackupHandler(c *gin.Context) { return } opu.BackupSetting = pu.BackupSetting - if err := opu.SetupAutomaticBackup(); err != nil { + if err := opu.SetupAutomaticBackup(nil); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrSettingupBackup}) return } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index 4489ece..291aa56 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "net/http" "strconv" + "time" "github.com/RichardKnop/machinery/v1/tasks" "github.com/vwxyzjn/portwarden" @@ -15,8 +16,8 @@ import ( ) type BackupSetting struct { - FileNamePrefix string `json:"filename_prefix"` - Passphrase string `json:"passphrase"` + Passphrase string `json:"passphrase"` + BackupFrequencySeconds int `json:"backup_frequency_seconds"` } type DecryptBackupInfo struct { @@ -113,7 +114,7 @@ func (pu *PortwardenUser) LoginWithBitwarden() error { return nil } -func (pu *PortwardenUser) SetupAutomaticBackup() error { +func (pu *PortwardenUser) SetupAutomaticBackup(eta *time.Time) error { signature := &tasks.Signature{ Name: "BackupToGoogleDrive", Args: []tasks.Arg{ @@ -122,6 +123,8 @@ func (pu *PortwardenUser) SetupAutomaticBackup() error { Value: pu.Email, }, }, + ETA: eta, + RetryCount: web.MachineryRetryCount, } _, err := web.MachineryServer.SendTask(signature) if err != nil { diff --git a/web/worker/main.go b/web/worker/main.go index 63febc1..886d64f 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "time" "github.com/vwxyzjn/portwarden" "github.com/vwxyzjn/portwarden/web" @@ -35,5 +36,7 @@ func BackupToGoogleDrive(email string) error { if err != nil { return err } + eta := time.Now().UTC().Add(time.Second * time.Duration(pu.BackupSetting.BackupFrequencySeconds)) + pu.SetupAutomaticBackup(&eta) return nil } From cb55f5a1cf56d2066e6c3c33a9e029ed9b48d535 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 09:08:03 -0500 Subject: [PATCH 20/36] API-6 # Add mutex to make sure bw login and backup are not disturbed by other processes. --- web/common.go | 2 ++ web/scheduler/server/model.go | 2 ++ web/worker/main.go | 7 ++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/web/common.go b/web/common.go index 0a0350c..dcc88d1 100644 --- a/web/common.go +++ b/web/common.go @@ -5,6 +5,7 @@ import ( "log" "os" "path/filepath" + "sync" machinery "github.com/RichardKnop/machinery/v1" "github.com/RichardKnop/machinery/v1/config" @@ -25,6 +26,7 @@ var ( RedisClient *redis.Client MachineryServer *machinery.Server BITWARDENCLI_APPDATA_DIR string + GlobalMutex sync.Mutex ) func InitCommonVars() { diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index 291aa56..ec826e6 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -98,6 +98,8 @@ func (pu *PortwardenUser) CreateWithGoogle() error { } func (pu *PortwardenUser) LoginWithBitwarden() error { + web.GlobalMutex.Lock() + defer web.GlobalMutex.Unlock() opu := PortwardenUser{Email: pu.Email} err := opu.Get() if err != nil { diff --git a/web/worker/main.go b/web/worker/main.go index 886d64f..28f02a9 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -22,6 +22,8 @@ func main() { } func BackupToGoogleDrive(email string) error { + web.GlobalMutex.Lock() + defer web.GlobalMutex.Unlock() pu := server.PortwardenUser{Email: email} fmt.Println(pu.Email) err := pu.Get() @@ -37,6 +39,9 @@ func BackupToGoogleDrive(email string) error { return err } eta := time.Now().UTC().Add(time.Second * time.Duration(pu.BackupSetting.BackupFrequencySeconds)) - pu.SetupAutomaticBackup(&eta) + err = pu.SetupAutomaticBackup(&eta) + if err != nil { + return err + } return nil } From 416eab31ae506a8940a328ce5071afc51d9bf931 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 10:45:32 -0500 Subject: [PATCH 21/36] API-6 # Allow the user to cancel backup setup --- web/scheduler/server/controller.go | 40 +++++++++++++++++++++++++++--- web/scheduler/server/model.go | 8 ++++++ web/scheduler/server/server.go | 1 + 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index ac8d775..ddb8c57 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -16,6 +16,9 @@ const ( ErrGettingPortwardenUser = "error getting a portwarden user" ErrLoginWithBitwarden = "error logging in with Bitwarden" ErrSettingupBackup = "error setting up backup" + ErrBackupNotCancelled = "error cancelling back up" + + MsgSuccessfullyCancelledBackingUp = "successfully cancelled backup process" FrontEndBaseAddressTest = "http://localhost:8000/" FrontEndBaseAddressProd = "" @@ -28,8 +31,8 @@ func EncryptBackupHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrBindingFromGin}) return } - if err := pu.LoginWithBitwarden(); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) + if !pu.BackupSetting.WillSetupBackup { + c.JSON(http.StatusBadRequest, gin.H{"message": MsgSuccessfullyCancelledBackingUp}) return } opu.Email = pu.Email @@ -37,9 +40,14 @@ func EncryptBackupHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) return } + if err := pu.LoginWithBitwarden(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) + return + } + opu.BackupSetting = pu.BackupSetting if err := opu.SetupAutomaticBackup(nil); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrSettingupBackup}) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": MsgSuccessfullyCancelledBackingUp}) return } if err := opu.Set(); err != nil { @@ -48,11 +56,35 @@ func EncryptBackupHandler(c *gin.Context) { } } +func CancelEncryptBackupHandler(c *gin.Context) { + var pu PortwardenUser + var opu PortwardenUser + if err := c.ShouldBindJSON(&pu); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrBindingFromGin}) + return + } + if pu.BackupSetting.WillSetupBackup { + c.JSON(http.StatusBadRequest, gin.H{"error": "", "message": ErrBackupNotCancelled}) + return + } + opu.Email = pu.Email + if err := opu.Get(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) + return + } + opu.BackupSetting = pu.BackupSetting + if err := opu.Set(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) + return + } + c.JSON(http.StatusOK, gin.H{"message": MsgSuccessfullyCancelledBackingUp}) +} + //TODO: GoogleDriveHandler() will return Json with the google login url // Not sure if it's supposed to call UploadFile() directly func (ps *PortwardenServer) GetGoogleDriveLoginURLHandler(c *gin.Context) { c.JSON(200, gin.H{ - "login_url": ps.GoogleDriveAppConfig.AuthCodeURL("state-token", oauth2.AccessTypeOffline, oauth2.ApprovalForce), + "login_url": web.GoogleDriveAppConfig.AuthCodeURL("state-token", oauth2.AccessTypeOffline, oauth2.ApprovalForce), }) return } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index ec826e6..07a9c46 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -15,9 +15,14 @@ import ( "golang.org/x/oauth2" ) +const ( + ErrWillNotSetupBackupByUser = "err the user stopped backing up" +) + type BackupSetting struct { Passphrase string `json:"passphrase"` BackupFrequencySeconds int `json:"backup_frequency_seconds"` + WillSetupBackup bool `json:"will_setup_backup"` } type DecryptBackupInfo struct { @@ -117,6 +122,9 @@ func (pu *PortwardenUser) LoginWithBitwarden() error { } func (pu *PortwardenUser) SetupAutomaticBackup(eta *time.Time) error { + if !pu.BackupSetting.WillSetupBackup { + return errors.New(ErrWillNotSetupBackupByUser) + } signature := &tasks.Signature{ Name: "BackupToGoogleDrive", Args: []tasks.Arg{ diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 735c280..388a72f 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -36,6 +36,7 @@ func (ps *PortwardenServer) Run() { c.JSON(200, "success") }) ps.Router.POST("/encrypt", EncryptBackupHandler) + ps.Router.POST("/encrypt/cancel", CancelEncryptBackupHandler) ps.Router.Run(":" + strconv.Itoa(ps.Port)) } From f8e8101bceb7395d1886c4badd887e38b47e802c Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 10:46:03 -0500 Subject: [PATCH 22/36] API-6 # Update access token --- web/scheduler/server/google_drive.go | 18 ++++++++++++------ web/worker/main.go | 12 +++++++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/web/scheduler/server/google_drive.go b/web/scheduler/server/google_drive.go index 8c12de9..7f75959 100644 --- a/web/scheduler/server/google_drive.go +++ b/web/scheduler/server/google_drive.go @@ -93,17 +93,23 @@ func SaveToken(file string, token *oauth2.Token) { // UploadFile upload the fileBytes to Google Drive's portwarden folder // https://gist.github.com/tzmartin/f5732091783752660b671c20479f519a -func UploadFile(fileBytes []byte, token *oauth2.Token) error { - client := web.GoogleDriveAppConfig.Client(oauth2.NoContext, token) +func UploadFile(fileBytes []byte, token *oauth2.Token) (*oauth2.Token, error) { + // Get updated access token + tokenSource := web.GoogleDriveAppConfig.TokenSource(oauth2.NoContext, token) + newToken, err := tokenSource.Token() + if err != nil { + return nil, err + } + client := web.GoogleDriveAppConfig.Client(oauth2.NoContext, newToken) srv, err := drive.New(client) if err != nil { - return err + return nil, err } mimeType := http.DetectContentType(fileBytes) parentId, err := GetOrCreateFolder(srv, web.PortwardenGoogleDriveBackupFolderName) if err != nil { - return err + return nil, err } f := &drive.File{Title: time.Now().Format("01-02-2006") + ".portwarden", MimeType: mimeType} @@ -114,10 +120,10 @@ func UploadFile(fileBytes []byte, token *oauth2.Token) error { _, err = srv.Files.Insert(f).Media(bytes.NewReader(fileBytes)).Do() if err != nil { - return err + return nil, err } - return nil + return newToken, nil } func GetOrCreateFolder(srv *drive.Service, folderName string) (string, error) { diff --git a/web/worker/main.go b/web/worker/main.go index 28f02a9..5b7a8f9 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "time" "github.com/vwxyzjn/portwarden" @@ -25,7 +24,6 @@ func BackupToGoogleDrive(email string) error { web.GlobalMutex.Lock() defer web.GlobalMutex.Unlock() pu := server.PortwardenUser{Email: email} - fmt.Println(pu.Email) err := pu.Get() if err != nil { return err @@ -34,12 +32,20 @@ func BackupToGoogleDrive(email string) error { if err != nil { return err } - err = server.UploadFile(encryptedData, pu.GoogleToken) + newToken, err := server.UploadFile(encryptedData, pu.GoogleToken) if err != nil { return err } + pu.GoogleToken = newToken eta := time.Now().UTC().Add(time.Second * time.Duration(pu.BackupSetting.BackupFrequencySeconds)) err = pu.SetupAutomaticBackup(&eta) + if err != nil { + if err.Error() != server.ErrWillNotSetupBackupByUser { + return err + } + } + // Update the access token + err = pu.Set() if err != nil { return err } From 5def1e5ca3574261175d187a022c6bb1ea89f17a Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 11:53:27 -0500 Subject: [PATCH 23/36] API-6 # All works now --- Gopkg.lock | 284 ++++++++++++++++++++++++++--- docker-compose.yml | 23 ++- web/scheduler/server/controller.go | 15 +- web/scheduler/server/model.go | 12 +- web/worker/main.go | 11 +- 5 files changed, 289 insertions(+), 56 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f55cecd..c3ce223 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,7 +3,15 @@ [[projects]] name = "cloud.google.com/go" - packages = ["compute/metadata","iam","internal/optional","internal/version","pubsub","pubsub/apiv1","pubsub/internal/distribution"] + packages = [ + "compute/metadata", + "iam", + "internal/optional", + "internal/version", + "pubsub", + "pubsub/apiv1", + "pubsub/internal/distribution" + ] revision = "74b12019e2aa53ec27882158f59192d7cd6d1998" version = "v0.33.1" @@ -15,7 +23,30 @@ [[projects]] name = "github.com/RichardKnop/machinery" - packages = ["v1","v1/backends/amqp","v1/backends/dynamodb","v1/backends/eager","v1/backends/iface","v1/backends/memcache","v1/backends/mongo","v1/backends/redis","v1/backends/result","v1/brokers/amqp","v1/brokers/eager","v1/brokers/errs","v1/brokers/gcppubsub","v1/brokers/iface","v1/brokers/redis","v1/brokers/sqs","v1/common","v1/config","v1/log","v1/retry","v1/tasks","v1/tracing"] + packages = [ + "v1", + "v1/backends/amqp", + "v1/backends/dynamodb", + "v1/backends/eager", + "v1/backends/iface", + "v1/backends/memcache", + "v1/backends/mongo", + "v1/backends/redis", + "v1/backends/result", + "v1/brokers/amqp", + "v1/brokers/eager", + "v1/brokers/errs", + "v1/brokers/gcppubsub", + "v1/brokers/iface", + "v1/brokers/redis", + "v1/brokers/sqs", + "v1/common", + "v1/config", + "v1/log", + "v1/retry", + "v1/tasks", + "v1/tracing" + ] revision = "21f6dd0def9983b08c773554c89f64b0176da3a2" version = "v1.5.3" @@ -25,15 +56,46 @@ revision = "54b27db64c180751e7049321a2e109ff220ccdcc" version = "v1.2.0" -[[projects]] - name = "github.com/antonholmquist/jason" - packages = ["."] - revision = "c23cef7eaa75a6a5b8810120e167bd590d8fd2ab" - version = "v1.0.0" - [[projects]] name = "github.com/aws/aws-sdk-go" - packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/crr","aws/csm","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/ini","internal/sdkio","internal/sdkrand","internal/sdkuri","internal/shareddefaults","private/protocol","private/protocol/json/jsonutil","private/protocol/jsonrpc","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/xml/xmlutil","service/dynamodb","service/dynamodb/dynamodbattribute","service/dynamodb/dynamodbiface","service/sqs","service/sqs/sqsiface","service/sts"] + packages = [ + "aws", + "aws/awserr", + "aws/awsutil", + "aws/client", + "aws/client/metadata", + "aws/corehandlers", + "aws/credentials", + "aws/credentials/ec2rolecreds", + "aws/credentials/endpointcreds", + "aws/credentials/stscreds", + "aws/crr", + "aws/csm", + "aws/defaults", + "aws/ec2metadata", + "aws/endpoints", + "aws/request", + "aws/session", + "aws/signer/v4", + "internal/ini", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", + "internal/shareddefaults", + "private/protocol", + "private/protocol/json/jsonutil", + "private/protocol/jsonrpc", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/xml/xmlutil", + "service/dynamodb", + "service/dynamodb/dynamodbattribute", + "service/dynamodb/dynamodbiface", + "service/sqs", + "service/sqs/sqsiface", + "service/sts" + ] revision = "08df30d135d32f1eb21bb7754eb042aeca521a4b" version = "v1.15.84" @@ -52,7 +114,14 @@ [[projects]] branch = "master" name = "github.com/dsnet/compress" - packages = [".","bzip2","bzip2/internal/sais","internal","internal/errors","internal/prefix"] + packages = [ + ".", + "bzip2", + "bzip2/internal/sais", + "internal", + "internal/errors", + "internal/prefix" + ] revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" [[projects]] @@ -69,19 +138,41 @@ [[projects]] name = "github.com/gin-gonic/gin" - packages = [".","binding","json","render"] + packages = [ + ".", + "binding", + "json", + "render" + ] revision = "b869fe1415e4b9eb52f247441830d502aece2d4d" version = "v1.3.0" [[projects]] name = "github.com/go-redis/redis" - packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto","internal/singleflight","internal/util"] + packages = [ + ".", + "internal", + "internal/consistenthash", + "internal/hashtag", + "internal/pool", + "internal/proto", + "internal/singleflight", + "internal/util" + ] revision = "b3d9bf10f6666b2ee5100a6f3f84f4caf3b4e37d" version = "v6.14.2" [[projects]] name = "github.com/golang/protobuf" - packages = ["proto","protoc-gen-go/descriptor","ptypes","ptypes/any","ptypes/duration","ptypes/empty","ptypes/timestamp"] + packages = [ + "proto", + "protoc-gen-go/descriptor", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/empty", + "ptypes/timestamp" + ] revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" @@ -93,7 +184,10 @@ [[projects]] name = "github.com/gomodule/redigo" - packages = ["internal","redis"] + packages = [ + "internal", + "redis" + ] revision = "9c11da706d9b7902c6da69c592f75637793fe121" version = "v2.0.0" @@ -109,6 +203,12 @@ revision = "b001040cd31805261cbd978842099e326dfa857b" version = "v2.0.2" +[[projects]] + name = "github.com/imdario/mergo" + packages = ["."] + revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" + version = "v0.3.6" + [[projects]] name = "github.com/jmespath/go-jmespath" packages = ["."] @@ -158,13 +258,20 @@ [[projects]] name = "github.com/opentracing/opentracing-go" - packages = [".","ext","log"] + packages = [ + ".", + "ext", + "log" + ] revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" [[projects]] name = "github.com/pierrec/lz4" - packages = [".","internal/xxh32"] + packages = [ + ".", + "internal/xxh32" + ] revision = "635575b42742856941dbc767b44905bb9ba083f6" version = "v2.0.7" @@ -188,13 +295,34 @@ [[projects]] name = "github.com/ulikunitz/xz" - packages = [".","internal/hash","internal/xlog","lzma"] + packages = [ + ".", + "internal/hash", + "internal/xlog", + "lzma" + ] revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" version = "v0.5.4" [[projects]] name = "go.opencensus.io" - packages = [".","exemplar","internal","internal/tagencoding","plugin/ocgrpc","plugin/ochttp","plugin/ochttp/propagation/b3","stats","stats/internal","stats/view","tag","trace","trace/internal","trace/propagation","trace/tracestate"] + packages = [ + ".", + "exemplar", + "internal", + "internal/tagencoding", + "plugin/ocgrpc", + "plugin/ochttp", + "plugin/ochttp/propagation/b3", + "stats", + "stats/internal", + "stats/view", + "tag", + "trace", + "trace/internal", + "trace/propagation", + "trace/tracestate" + ] revision = "b7bf3cdb64150a8c8c53b769fdeb2ba581bd4d4b" version = "v0.18.0" @@ -207,19 +335,37 @@ [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context","context/ctxhttp","http/httpguts","http2","http2/hpack","idna","internal/timeseries","trace"] + packages = [ + "context", + "context/ctxhttp", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace" + ] revision = "adae6a3d119ae4890b46832a2e88a95adc62b8e7" [[projects]] branch = "master" name = "golang.org/x/oauth2" - packages = [".","google","internal","jws","jwt"] + packages = [ + ".", + "google", + "internal", + "jws", + "jwt" + ] revision = "f42d05182288abf10faef86d16c0d07b8d40ea2d" [[projects]] branch = "master" name = "golang.org/x/sync" - packages = ["errgroup","semaphore"] + packages = [ + "errgroup", + "semaphore" + ] revision = "42b317875d0fa942474b76e1b46a6060d720ae6e" [[projects]] @@ -230,31 +376,107 @@ [[projects]] name = "golang.org/x/text" - packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" name = "google.golang.org/api" - packages = ["drive/v2","gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","support/bundler","transport","transport/grpc","transport/http","transport/http/internal/propagation"] + packages = [ + "drive/v2", + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "googleapi/transport", + "internal", + "iterator", + "option", + "support/bundler", + "transport", + "transport/grpc", + "transport/http", + "transport/http/internal/propagation" + ] revision = "83a9d304b1e613fc253e1e2710778642fe81af53" [[projects]] name = "google.golang.org/appengine" - packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/socket","internal/urlfetch","socket","urlfetch"] + packages = [ + ".", + "internal", + "internal/app_identity", + "internal/base", + "internal/datastore", + "internal/log", + "internal/modules", + "internal/remote_api", + "internal/socket", + "internal/urlfetch", + "socket", + "urlfetch" + ] revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" version = "v1.3.0" [[projects]] branch = "master" name = "google.golang.org/genproto" - packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/pubsub/v1","googleapis/rpc/status","protobuf/field_mask"] + packages = [ + "googleapis/api/annotations", + "googleapis/iam/v1", + "googleapis/pubsub/v1", + "googleapis/rpc/status", + "protobuf/field_mask" + ] revision = "31ac5d88444a9e7ad18077db9a165d793ad06a2e" [[projects]] name = "google.golang.org/grpc" - packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","credentials/oauth","encoding","encoding/proto","grpclog","internal","internal/backoff","internal/channelz","internal/envconfig","internal/grpcrand","internal/transport","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap"] + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "credentials/oauth", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap" + ] revision = "2e463a05d100327ca47ac218281906921038fd95" version = "v1.16.0" @@ -267,7 +489,13 @@ [[projects]] branch = "v2" name = "gopkg.in/mgo.v2" - packages = [".","bson","internal/json","internal/sasl","internal/scram"] + packages = [ + ".", + "bson", + "internal/json", + "internal/sasl", + "internal/scram" + ] revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5" [[projects]] @@ -285,6 +513,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d622028d481381214ed075bf271e699c86fc8f22690364f507e3d82def387ea8" + inputs-digest = "d7e60fff4e12704fe15d11a18730ac1a45c45013ff52c41872a5852c517a1199" solver-name = "gps-cdcl" solver-version = 1 diff --git a/docker-compose.yml b/docker-compose.yml index 6521f45..cfeb9d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-scheduler:0.1.1 + image: vwxyzjn/portwarden-base:1.1.0 stdin_open: true tty: true environment: @@ -12,8 +12,12 @@ services: - redis ports: - 5000:5000 - volumes: - - .:/go/src/github.com/vwxyzjn/portwarden + # volumes: + # - .:/go/src/github.com/vwxyzjn/portwarden + working_dir: + /go/src/github.com/vwxyzjn/portwarden/web/scheduler + command: + go run main.go redis: image: redis @@ -21,15 +25,22 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-scheduler:0.1.1 + image: vwxyzjn/portwarden-base:1.1.0 stdin_open: true tty: true environment: - BITWARDENCLI_APPDATA_DIR=/BitwardenCLI depends_on: - redis - volumes: - - .:/go/src/github.com/vwxyzjn/portwarden + deploy: + mode: replicated + replicas: 6 + # volumes: + # - .:/go/src/github.com/vwxyzjn/portwarden + working_dir: + /go/src/github.com/vwxyzjn/portwarden/web/worker + command: + go run main.go redis-commander: image: rediscommander/redis-commander:latest diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index ddb8c57..8c186c2 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -3,8 +3,8 @@ package server import ( "net/http" - "github.com/davecgh/go-spew/spew" "github.com/gin-gonic/gin" + "github.com/imdario/mergo" "github.com/vwxyzjn/portwarden/web" "golang.org/x/oauth2" ) @@ -31,26 +31,21 @@ func EncryptBackupHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrBindingFromGin}) return } - if !pu.BackupSetting.WillSetupBackup { - c.JSON(http.StatusBadRequest, gin.H{"message": MsgSuccessfullyCancelledBackingUp}) - return - } opu.Email = pu.Email if err := opu.Get(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) return } + mergo.Merge(&pu, opu) if err := pu.LoginWithBitwarden(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrLoginWithBitwarden}) return } - - opu.BackupSetting = pu.BackupSetting - if err := opu.SetupAutomaticBackup(nil); err != nil { + if err := pu.SetupAutomaticBackup(nil); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": MsgSuccessfullyCancelledBackingUp}) return } - if err := opu.Set(); err != nil { + if err := pu.Set(); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) return } @@ -106,8 +101,6 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) return } - - spew.Dump(pu) c.Redirect(http.StatusMovedPermanently, FrontEndBaseAddressTest+"?access_token="+pu.GoogleToken.AccessToken) return } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index 07a9c46..8a08dc5 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -105,16 +105,8 @@ func (pu *PortwardenUser) CreateWithGoogle() error { func (pu *PortwardenUser) LoginWithBitwarden() error { web.GlobalMutex.Lock() defer web.GlobalMutex.Unlock() - opu := PortwardenUser{Email: pu.Email} - err := opu.Get() - if err != nil { - return err - } - opu.BitwardenSessionKey, opu.BitwardenDataJSON, err = portwarden.BWLoginGetSessionKeyAndDataJSON(pu.BitwardenLoginCredentials, web.BITWARDENCLI_APPDATA_DIR) - if err != nil { - return err - } - err = opu.Set() + var err error + pu.BitwardenSessionKey, pu.BitwardenDataJSON, err = portwarden.BWLoginGetSessionKeyAndDataJSON(pu.BitwardenLoginCredentials, web.BITWARDENCLI_APPDATA_DIR) if err != nil { return err } diff --git a/web/worker/main.go b/web/worker/main.go index 5b7a8f9..0d55713 100644 --- a/web/worker/main.go +++ b/web/worker/main.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + "os" "time" "github.com/vwxyzjn/portwarden" @@ -23,8 +25,13 @@ func main() { func BackupToGoogleDrive(email string) error { web.GlobalMutex.Lock() defer web.GlobalMutex.Unlock() + name, err := os.Hostname() + if err != nil { + return err + } + fmt.Println("BackupToGoogleDrive called from worker:", name) pu := server.PortwardenUser{Email: email} - err := pu.Get() + err = pu.Get() if err != nil { return err } @@ -42,6 +49,8 @@ func BackupToGoogleDrive(email string) error { if err != nil { if err.Error() != server.ErrWillNotSetupBackupByUser { return err + } else { + fmt.Printf("user %v cancelled backup", pu.Email) } } // Update the access token From 34f7fe5254394aa3092fa6ed2495ee22d4b2e490 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 11:54:32 -0500 Subject: [PATCH 24/36] API-6 # Rename --- docker-compose.yml => docker-compose.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker-compose.yml => docker-compose.yaml (100%) diff --git a/docker-compose.yml b/docker-compose.yaml similarity index 100% rename from docker-compose.yml rename to docker-compose.yaml From d5e47bb907b7e09a428c7ddf062b713a211a4d97 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 15:32:27 -0500 Subject: [PATCH 25/36] API-6 # Temp change --- Dockerfile | 16 +++++++++++++++- docker-compose.yaml | 10 +++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index dcc893c..2b39832 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" # Setup work directory COPY . /go/src/github.com/vwxyzjn/portwarden -WORKDIR /go/src/github.com/vwxyzjn/portwarden/web/scheduler +WORKDIR /go/src/github.com/vwxyzjn/portwarden # Install Go Dep RUN wget -q https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 @@ -28,5 +28,19 @@ RUN chmod +x /usr/bin/bw # Notice git is the dependency for running dep RUN cd /go/src/github.com/vwxyzjn/portwarden && dep ensure --vendor-only +RUN go build /go/src/github.com/vwxyzjn/portwarden/web/worker/main.go && mv ./main /worker +RUN go build /go/src/github.com/vwxyzjn/portwarden/web/scheduler/main.go && mv ./main /scheduler + # Ready to run EXPOSE 5000 + + +FROM debian:stretch-20181112 +COPY --from=0 /usr/bin/bw /usr/bin/bw +COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler +COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker +COPY --from=0 /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json +RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler +RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/worker/worker +WORKDIR /go/src/github.com/vwxyzjn/portwarden +EXPOSE 5000 diff --git a/docker-compose.yaml b/docker-compose.yaml index cfeb9d2..f230b55 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-base:1.1.0 + image: vwxyzjn/portwarden-base:1.2.0 stdin_open: true tty: true environment: @@ -17,7 +17,7 @@ services: working_dir: /go/src/github.com/vwxyzjn/portwarden/web/scheduler command: - go run main.go + ./scheduler redis: image: redis @@ -25,7 +25,7 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-base:1.1.0 + image: vwxyzjn/portwarden-base:1.2.0 stdin_open: true tty: true environment: @@ -34,13 +34,13 @@ services: - redis deploy: mode: replicated - replicas: 6 + replicas: 2 # volumes: # - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/worker command: - go run main.go + ./worker redis-commander: image: rediscommander/redis-commander:latest From 7d03cd3784236fd420681661b49f4946ecee463f Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 15:34:13 -0500 Subject: [PATCH 26/36] API-6 # Minor update --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b39832..319fe26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ FROM ubuntu:latest # Install Go -RUN apt-get update -RUN apt-get install -y wget git gcc unzip +RUN apt-get update && apt-get install -y wget git gcc unzip RUN wget -q -P /tmp https://dl.google.com/go/go1.11.2.linux-amd64.tar.gz RUN tar -C /usr/local -xzf /tmp/go1.11.2.linux-amd64.tar.gz RUN rm /tmp/go1.11.2.linux-amd64.tar.gz From efd440bd82d497a710ab63709bf7f8d6ae33dd94 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Sun, 2 Dec 2018 15:53:22 -0500 Subject: [PATCH 27/36] API-6 # Setup dev and deploy docker files --- Dockerfile | 14 ------ Dockerfile.Build | 45 +++++++++++++++++++ docker-compose.build.yaml | 51 +++++++++++++++++++++ docker-compose.yaml | 16 +++---- k8s.yaml | 93 --------------------------------------- 5 files changed, 104 insertions(+), 115 deletions(-) create mode 100644 Dockerfile.Build create mode 100644 docker-compose.build.yaml delete mode 100644 k8s.yaml diff --git a/Dockerfile b/Dockerfile index 319fe26..5f53ae0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,19 +27,5 @@ RUN chmod +x /usr/bin/bw # Notice git is the dependency for running dep RUN cd /go/src/github.com/vwxyzjn/portwarden && dep ensure --vendor-only -RUN go build /go/src/github.com/vwxyzjn/portwarden/web/worker/main.go && mv ./main /worker -RUN go build /go/src/github.com/vwxyzjn/portwarden/web/scheduler/main.go && mv ./main /scheduler - # Ready to run EXPOSE 5000 - - -FROM debian:stretch-20181112 -COPY --from=0 /usr/bin/bw /usr/bin/bw -COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler -COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker -COPY --from=0 /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json -RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler -RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/worker/worker -WORKDIR /go/src/github.com/vwxyzjn/portwarden -EXPOSE 5000 diff --git a/Dockerfile.Build b/Dockerfile.Build new file mode 100644 index 0000000..319fe26 --- /dev/null +++ b/Dockerfile.Build @@ -0,0 +1,45 @@ +FROM ubuntu:latest + +# Install Go +RUN apt-get update && apt-get install -y wget git gcc unzip +RUN wget -q -P /tmp https://dl.google.com/go/go1.11.2.linux-amd64.tar.gz +RUN tar -C /usr/local -xzf /tmp/go1.11.2.linux-amd64.tar.gz +RUN rm /tmp/go1.11.2.linux-amd64.tar.gz +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +# Setup work directory +COPY . /go/src/github.com/vwxyzjn/portwarden +WORKDIR /go/src/github.com/vwxyzjn/portwarden + +# Install Go Dep +RUN wget -q https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 +RUN mv dep-linux-amd64 /usr/bin/dep +RUN chmod +x /usr/bin/dep + +# Install Bitwarden CLI +RUN wget -q https://github.com/bitwarden/cli/releases/download/v1.6.0/bw-linux-1.6.0.zip +RUN unzip bw-linux-1.6.0.zip -d /usr/bin/ +RUN chmod +x /usr/bin/bw + +# Run dep +# Notice git is the dependency for running dep +RUN cd /go/src/github.com/vwxyzjn/portwarden && dep ensure --vendor-only + +RUN go build /go/src/github.com/vwxyzjn/portwarden/web/worker/main.go && mv ./main /worker +RUN go build /go/src/github.com/vwxyzjn/portwarden/web/scheduler/main.go && mv ./main /scheduler + +# Ready to run +EXPOSE 5000 + + +FROM debian:stretch-20181112 +COPY --from=0 /usr/bin/bw /usr/bin/bw +COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler +COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker +COPY --from=0 /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json /go/src/github.com/vwxyzjn/portwarden/web/portwardenCredentials.json +RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler +RUN chmod +x /go/src/github.com/vwxyzjn/portwarden/web/worker/worker +WORKDIR /go/src/github.com/vwxyzjn/portwarden +EXPOSE 5000 diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml new file mode 100644 index 0000000..b8d1cc2 --- /dev/null +++ b/docker-compose.build.yaml @@ -0,0 +1,51 @@ +version: '3' + +services: + + scheduler: + image: vwxyzjn/portwarden-base:1.3.0 + stdin_open: true + tty: true + environment: + - BITWARDENCLI_APPDATA_DIR=/BitwardenCLI + depends_on: + - redis + ports: + - 5000:5000 + # volumes: + # - .:/go/src/github.com/vwxyzjn/portwarden + working_dir: + /go/src/github.com/vwxyzjn/portwarden/web/scheduler + command: + ./scheduler + + redis: + image: redis + ports: + - 6379:6379 + + worker: + image: vwxyzjn/portwarden-base:1.3.0 + stdin_open: true + tty: true + environment: + - BITWARDENCLI_APPDATA_DIR=/BitwardenCLI + depends_on: + - redis + deploy: + mode: replicated + replicas: 2 + # volumes: + # - .:/go/src/github.com/vwxyzjn/portwarden + working_dir: + /go/src/github.com/vwxyzjn/portwarden/web/worker + command: + ./worker + + redis-commander: + image: rediscommander/redis-commander:latest + restart: always + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8081:8081" diff --git a/docker-compose.yaml b/docker-compose.yaml index f230b55..9b0ee37 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-base:1.2.0 + image: vwxyzjn/portwarden-base:1.1.0 stdin_open: true tty: true environment: @@ -12,12 +12,12 @@ services: - redis ports: - 5000:5000 - # volumes: - # - .:/go/src/github.com/vwxyzjn/portwarden + volumes: + - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/scheduler command: - ./scheduler + go run main.go redis: image: redis @@ -25,7 +25,7 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-base:1.2.0 + image: vwxyzjn/portwarden-base:1.1.0 stdin_open: true tty: true environment: @@ -35,12 +35,12 @@ services: deploy: mode: replicated replicas: 2 - # volumes: - # - .:/go/src/github.com/vwxyzjn/portwarden + volumes: + - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/worker command: - ./worker + go run main.go redis-commander: image: rediscommander/redis-commander:latest diff --git a/k8s.yaml b/k8s.yaml deleted file mode 100644 index 5012440..0000000 --- a/k8s.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: portwarden - - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: portwarden - name: portwarden-scheduler - labels: - app: portwarden-scheduler -spec: - selector: - matchLabels: - app: portwarden-scheduler - tier: backend - # replicas: 7 - template: - metadata: - labels: - app: portwarden-scheduler - tier: backend - spec: - containers: - - name: portwarden-scheduler - image: vwxyzjn/portwarden-scheduler:0.0.1 - ports: - - name: http - containerPort: 5000 - stdin: true - tty: true -# # START of dev related lines, local mount doesn't work on windows -# volumeMounts: -# - mountPath: /go/src/github.com/vwxyzjn/portwarden -# name: dev-volume -# volumes: -# - name: dev-volume -# hostPath: -# path: /data -# # END of dev related lines ---- -kind: Service -apiVersion: v1 -metadata: - namespace: portwarden - name: portwarden-scheduler -spec: - selector: - app: portwarden-scheduler - tier: backend - ports: - - port: 5000 - targetPort: 5000 - - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: portwarden - name: redis -spec: - selector: - matchLabels: - app: redis - tier: backend - template: - metadata: - labels: - app: redis - tier: backend - spec: - containers: - - name: redis - image: redis - ports: - - containerPort: 6379 ---- -kind: Service -apiVersion: v1 -metadata: - namespace: portwarden - name: redis -spec: - selector: - app: redis - tier: backend - ports: - - port: 6379 - targetPort: 6379 From cb656be820742b1ebadccacf3417e2807313d1ba Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 02:35:43 -0500 Subject: [PATCH 28/36] API-6 # Update the docker-compose.build.yaml --- docker-compose.build.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index b8d1cc2..c2019cd 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -18,6 +18,9 @@ services: /go/src/github.com/vwxyzjn/portwarden/web/scheduler command: ./scheduler + labels: + kompose.service.expose: "true" + kompose.service.type: "loadbalancer" redis: image: redis @@ -49,3 +52,6 @@ services: - REDIS_HOSTS=local:redis:6379 ports: - "8081:8081" + labels: + kompose.service.expose: "true" + kompose.service.type: "loadbalancer" From 7edc0d862f4014ae1fdccb63800c0552bd5ba726 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 10:15:24 -0500 Subject: [PATCH 29/36] API-6 # Handle exisiting users --- docker-compose.yaml | 4 +-- web/scheduler/server/controller.go | 26 ++++++++++++--- web/scheduler/server/model.go | 52 +++++++++++++++++------------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9b0ee37..d4858df 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -16,8 +16,8 @@ services: - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/scheduler - command: - go run main.go + # command: + # go run main.go redis: image: redis diff --git a/web/scheduler/server/controller.go b/web/scheduler/server/controller.go index 8c186c2..ec3f231 100644 --- a/web/scheduler/server/controller.go +++ b/web/scheduler/server/controller.go @@ -2,8 +2,10 @@ package server import ( "net/http" + "strconv" "github.com/gin-gonic/gin" + "github.com/go-redis/redis" "github.com/imdario/mergo" "github.com/vwxyzjn/portwarden/web" "golang.org/x/oauth2" @@ -95,13 +97,29 @@ func (ps *PortwardenServer) GetGoogleDriveLoginHandler(c *gin.Context) { c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) return } - pu := &PortwardenUser{GoogleToken: tok} - err = pu.CreateWithGoogle() + gui, err := RetrieveUserEmail(tok) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) + c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error(), "message": "Login failure"}) return } - c.Redirect(http.StatusMovedPermanently, FrontEndBaseAddressTest+"?access_token="+pu.GoogleToken.AccessToken) + pu := &PortwardenUser{Email: gui.Email, GoogleToken: tok} + err = pu.Get() + if err != nil { + if err != redis.Nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrGettingPortwardenUser}) + return + } + // Create a user + err = pu.CreateWithGoogle() + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error(), "message": ErrCreatingPortwardenUser}) + return + } + c.Redirect(http.StatusMovedPermanently, FrontEndBaseAddressTest+"home/"+"?access_token="+pu.GoogleToken.AccessToken+"&email="+pu.Email+"&will_setup_backup="+strconv.FormatBool(false)) + return + } + // Using info from exisiting user + c.Redirect(http.StatusMovedPermanently, FrontEndBaseAddressTest+"home/"+"?access_token="+tok.AccessToken+"&email="+pu.Email+"&will_setup_backup="+strconv.FormatBool(pu.BackupSetting.WillSetupBackup)) return } diff --git a/web/scheduler/server/model.go b/web/scheduler/server/model.go index 8a08dc5..e78e6d1 100644 --- a/web/scheduler/server/model.go +++ b/web/scheduler/server/model.go @@ -69,31 +69,11 @@ type GoogleUserInfo struct { } func (pu *PortwardenUser) CreateWithGoogle() error { - postURL := "https://www.googleapis.com/oauth2/v2/userinfo" - request, err := http.NewRequest("GET", postURL, nil) + var err error + pu.GoogleUserInfo, err = RetrieveUserEmail(pu.GoogleToken) if err != nil { return err } - request.Header.Add("Host", "www.googleapis.com") - request.Header.Add("Authorization", "Bearer "+pu.GoogleToken.AccessToken) - request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) - - // For debugging - //fmt.Println(request) - GoogleDriveClient := web.GoogleDriveAppConfig.Client(oauth2.NoContext, pu.GoogleToken) - response, err := GoogleDriveClient.Do(request) - if err != nil { - return err - } - - defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return err - } - if err := json.Unmarshal(body, &pu.GoogleUserInfo); err != nil { - return err - } pu.Email = pu.GoogleUserInfo.Email err = pu.Set() if err != nil { @@ -179,3 +159,31 @@ func VerifyGoogleAccessToekn(access_token string) (bool, error) { } return true, nil } + +func RetrieveUserEmail(token *oauth2.Token) (GoogleUserInfo, error) { + var gui GoogleUserInfo + postURL := "https://www.googleapis.com/oauth2/v2/userinfo" + request, err := http.NewRequest("GET", postURL, nil) + if err != nil { + return gui, err + } + request.Header.Add("Host", "www.googleapis.com") + request.Header.Add("Authorization", "Bearer "+token.AccessToken) + request.Header.Add("Content-Length", strconv.FormatInt(request.ContentLength, 10)) + + GoogleDriveClient := web.GoogleDriveAppConfig.Client(oauth2.NoContext, token) + response, err := GoogleDriveClient.Do(request) + if err != nil { + return gui, err + } + + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return gui, err + } + if err := json.Unmarshal(body, &gui); err != nil { + return gui, err + } + return gui, nil +} From 8bbf912ca642791df4d881d99ccf3dd01c7e338e Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 10:42:47 -0500 Subject: [PATCH 30/36] API-6 # put the build file in a different folder --- .../docker-compose.build.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) rename docker-compose.build.yaml => k8s/docker-compose.build.yaml (82%) diff --git a/docker-compose.build.yaml b/k8s/docker-compose.build.yaml similarity index 82% rename from docker-compose.build.yaml rename to k8s/docker-compose.build.yaml index c2019cd..602dd73 100644 --- a/docker-compose.build.yaml +++ b/k8s/docker-compose.build.yaml @@ -3,7 +3,7 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-base:1.3.0 + image: vwxyzjn/portwarden-base:1.5.0 stdin_open: true tty: true environment: @@ -12,8 +12,6 @@ services: - redis ports: - 5000:5000 - # volumes: - # - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/scheduler command: @@ -28,7 +26,7 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-base:1.3.0 + image: vwxyzjn/portwarden-base:1.5.0 stdin_open: true tty: true environment: @@ -38,8 +36,6 @@ services: deploy: mode: replicated replicas: 2 - # volumes: - # - .:/go/src/github.com/vwxyzjn/portwarden working_dir: /go/src/github.com/vwxyzjn/portwarden/web/worker command: From e3306a35954251343424f5bfff91ac06d823f7fc Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 10:44:13 -0500 Subject: [PATCH 31/36] API-6 # Update dockerfile --- Dockerfile.Build | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.Build b/Dockerfile.Build index 319fe26..1939027 100644 --- a/Dockerfile.Build +++ b/Dockerfile.Build @@ -35,6 +35,7 @@ EXPOSE 5000 FROM debian:stretch-20181112 +apt-get install --no-cache ca-certificates openssl COPY --from=0 /usr/bin/bw /usr/bin/bw COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker From 18bd88a4133c31e22892a598587859a919f10eb4 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 10:44:54 -0500 Subject: [PATCH 32/36] API-6 # Minor Update --- Dockerfile.Build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.Build b/Dockerfile.Build index 1939027..9f0b32b 100644 --- a/Dockerfile.Build +++ b/Dockerfile.Build @@ -35,7 +35,7 @@ EXPOSE 5000 FROM debian:stretch-20181112 -apt-get install --no-cache ca-certificates openssl +RUN apt-get install --no-cache ca-certificates openssl COPY --from=0 /usr/bin/bw /usr/bin/bw COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker From b72429dc12171f84c67a710785177ac504f93efe Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 17:46:05 -0500 Subject: [PATCH 33/36] API-6 # Use middleware that is friendly to OPTIONS requests --- web/scheduler/server/middleware.go | 17 +++++++++++++++++ web/scheduler/server/server.go | 6 ++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/web/scheduler/server/middleware.go b/web/scheduler/server/middleware.go index d1ca547..6972827 100644 --- a/web/scheduler/server/middleware.go +++ b/web/scheduler/server/middleware.go @@ -31,3 +31,20 @@ func TokenAuthMiddleware() gin.HandlerFunc { c.Next() } } + +func CORSMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Max-Age", "86400") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, UPDATE") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization") + c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Length") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(200) + } else { + c.Next() + } + } +} diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index 388a72f..c8040ef 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -1,10 +1,8 @@ package server import ( - "net/http" "strconv" - "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "golang.org/x/net/context" "golang.org/x/oauth2" @@ -20,10 +18,10 @@ type PortwardenServer struct { func (ps *PortwardenServer) Run() { ps.Router = gin.Default() - ps.Router.Use(cors.Default()) + ps.Router.Use(CORSMiddleware()) ps.Router.GET("/", func(c *gin.Context) { - http.ServeFile(c.Writer, c.Request, "index.html") + c.JSON(200, "Welcome to Portwarden API") }) ps.Router.POST("/decrypt", DecryptBackupHandler) From d691dbabb21fa6863cdfb1a78626b24cad79e848 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 18:07:49 -0500 Subject: [PATCH 34/36] API-6 # Make sure the container have public certificate so that it can make calls to google services. --- Dockerfile.Build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.Build b/Dockerfile.Build index 9f0b32b..786309f 100644 --- a/Dockerfile.Build +++ b/Dockerfile.Build @@ -35,7 +35,7 @@ EXPOSE 5000 FROM debian:stretch-20181112 -RUN apt-get install --no-cache ca-certificates openssl +RUN apt-get update && apt-get install -y ca-certificates openssl COPY --from=0 /usr/bin/bw /usr/bin/bw COPY --from=0 /scheduler /go/src/github.com/vwxyzjn/portwarden/web/scheduler/scheduler COPY --from=0 /worker /go/src/github.com/vwxyzjn/portwarden/web/worker/worker From 66c2f6c88cc8407f426eab3299bfa4820fd98f93 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 18:32:56 -0500 Subject: [PATCH 35/36] API-6 # Update the lock file --- Gopkg.lock | 8 +------- web/scheduler/server/server.go | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c3ce223..ce80e4e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -124,12 +124,6 @@ ] revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" -[[projects]] - name = "github.com/gin-contrib/cors" - packages = ["."] - revision = "cf4846e6a636a76237a28d9286f163c132e841bc" - version = "v1.2" - [[projects]] branch = "master" name = "github.com/gin-contrib/sse" @@ -513,6 +507,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d7e60fff4e12704fe15d11a18730ac1a45c45013ff52c41872a5852c517a1199" + inputs-digest = "b0982d243cf6580588055e8aa56320744ba316df6e3a49ca69fd19ec8211300c" solver-name = "gps-cdcl" solver-version = 1 diff --git a/web/scheduler/server/server.go b/web/scheduler/server/server.go index c8040ef..276d4ba 100644 --- a/web/scheduler/server/server.go +++ b/web/scheduler/server/server.go @@ -3,6 +3,8 @@ package server import ( "strconv" + "github.com/davecgh/go-spew/spew" + "github.com/gin-gonic/gin" "golang.org/x/net/context" "golang.org/x/oauth2" @@ -17,6 +19,7 @@ type PortwardenServer struct { } func (ps *PortwardenServer) Run() { + spew.Dump("Scheduler Server Started") ps.Router = gin.Default() ps.Router.Use(CORSMiddleware()) From 32ad1a58fceb8ff1d902fdfa48e2a7e2e6bb85d1 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Mon, 3 Dec 2018 23:53:39 -0500 Subject: [PATCH 36/36] API-6 # Add k8s definitions --- k8s/docker-compose.build.yaml | 4 +-- k8s/kompose/redis-commander-deployment.yaml | 32 ++++++++++++++++++ k8s/kompose/redis-commander-ingress.yaml | 16 +++++++++ k8s/kompose/redis-commander-service.yaml | 22 ++++++++++++ k8s/kompose/redis-deployment.yaml | 27 +++++++++++++++ k8s/kompose/redis-service.yaml | 19 +++++++++++ k8s/kompose/scheduler-deployment.yaml | 37 +++++++++++++++++++++ k8s/kompose/scheduler-ingress.yaml | 16 +++++++++ k8s/kompose/scheduler-service.yaml | 23 +++++++++++++ k8s/kompose/worker-deployment.yaml | 33 ++++++++++++++++++ 10 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 k8s/kompose/redis-commander-deployment.yaml create mode 100644 k8s/kompose/redis-commander-ingress.yaml create mode 100644 k8s/kompose/redis-commander-service.yaml create mode 100644 k8s/kompose/redis-deployment.yaml create mode 100644 k8s/kompose/redis-service.yaml create mode 100644 k8s/kompose/scheduler-deployment.yaml create mode 100644 k8s/kompose/scheduler-ingress.yaml create mode 100644 k8s/kompose/scheduler-service.yaml create mode 100644 k8s/kompose/worker-deployment.yaml diff --git a/k8s/docker-compose.build.yaml b/k8s/docker-compose.build.yaml index 602dd73..ca95822 100644 --- a/k8s/docker-compose.build.yaml +++ b/k8s/docker-compose.build.yaml @@ -3,7 +3,7 @@ version: '3' services: scheduler: - image: vwxyzjn/portwarden-base:1.5.0 + image: vwxyzjn/portwarden-base:1.6.0 stdin_open: true tty: true environment: @@ -26,7 +26,7 @@ services: - 6379:6379 worker: - image: vwxyzjn/portwarden-base:1.5.0 + image: vwxyzjn/portwarden-base:1.6.0 stdin_open: true tty: true environment: diff --git a/k8s/kompose/redis-commander-deployment.yaml b/k8s/kompose/redis-commander-deployment.yaml new file mode 100644 index 0000000..3a3fce4 --- /dev/null +++ b/k8s/kompose/redis-commander-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.service.expose: "true" + kompose.service.type: loadbalancer + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: redis-commander + name: redis-commander +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: redis-commander + spec: + containers: + - env: + - name: REDIS_HOSTS + value: local:redis:6379 + image: rediscommander/redis-commander:latest + name: redis-commander + ports: + - containerPort: 8081 + resources: {} + restartPolicy: Always +status: {} diff --git a/k8s/kompose/redis-commander-ingress.yaml b/k8s/kompose/redis-commander-ingress.yaml new file mode 100644 index 0000000..44511eb --- /dev/null +++ b/k8s/kompose/redis-commander-ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + creationTimestamp: null + labels: + io.kompose.service: redis-commander + name: redis-commander +spec: + rules: + - http: + paths: + - backend: + serviceName: redis-commander + servicePort: 8081 +status: + loadBalancer: {} diff --git a/k8s/kompose/redis-commander-service.yaml b/k8s/kompose/redis-commander-service.yaml new file mode 100644 index 0000000..ce8d0da --- /dev/null +++ b/k8s/kompose/redis-commander-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.service.expose: "true" + kompose.service.type: loadbalancer + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: redis-commander + name: redis-commander +spec: + ports: + - name: "8081" + port: 8081 + targetPort: 8081 + selector: + io.kompose.service: redis-commander + type: LoadBalancer +status: + loadBalancer: {} diff --git a/k8s/kompose/redis-deployment.yaml b/k8s/kompose/redis-deployment.yaml new file mode 100644 index 0000000..772c583 --- /dev/null +++ b/k8s/kompose/redis-deployment.yaml @@ -0,0 +1,27 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: redis + name: redis +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: redis + spec: + containers: + - image: redis + name: redis + ports: + - containerPort: 6379 + resources: {} + restartPolicy: Always +status: {} diff --git a/k8s/kompose/redis-service.yaml b/k8s/kompose/redis-service.yaml new file mode 100644 index 0000000..9ded402 --- /dev/null +++ b/k8s/kompose/redis-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: redis + name: redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + io.kompose.service: redis +status: + loadBalancer: {} diff --git a/k8s/kompose/scheduler-deployment.yaml b/k8s/kompose/scheduler-deployment.yaml new file mode 100644 index 0000000..7ba9fba --- /dev/null +++ b/k8s/kompose/scheduler-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.service.expose: "true" + kompose.service.type: loadbalancer + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: scheduler + name: scheduler +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: scheduler + spec: + containers: + - args: + - ./scheduler + env: + - name: BITWARDENCLI_APPDATA_DIR + value: /BitwardenCLI + image: vwxyzjn/portwarden-base:1.6.0 + name: scheduler + ports: + - containerPort: 5000 + resources: {} + stdin: true + tty: true + workingDir: /go/src/github.com/vwxyzjn/portwarden/web/scheduler + restartPolicy: Always +status: {} diff --git a/k8s/kompose/scheduler-ingress.yaml b/k8s/kompose/scheduler-ingress.yaml new file mode 100644 index 0000000..ab56abc --- /dev/null +++ b/k8s/kompose/scheduler-ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + creationTimestamp: null + labels: + io.kompose.service: scheduler + name: scheduler +spec: + rules: + - http: + paths: + - backend: + serviceName: scheduler + servicePort: 5000 +status: + loadBalancer: {} diff --git a/k8s/kompose/scheduler-service.yaml b/k8s/kompose/scheduler-service.yaml new file mode 100644 index 0000000..19c1ec7 --- /dev/null +++ b/k8s/kompose/scheduler-service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.service.expose: "true" + kompose.service.type: loadbalancer + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: scheduler + name: scheduler +spec: + ports: + - name: "5000" + port: 5000 + targetPort: 5000 + nodePort: 32222 + selector: + io.kompose.service: scheduler + type: LoadBalancer +status: + loadBalancer: {} diff --git a/k8s/kompose/worker-deployment.yaml b/k8s/kompose/worker-deployment.yaml new file mode 100644 index 0000000..1f20e87 --- /dev/null +++ b/k8s/kompose/worker-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + kompose.cmd: C:\Go\bin\kompose.exe convert -f docker-compose.build.yaml + kompose.version: 1.17.0 (a74acad) + creationTimestamp: null + labels: + io.kompose.service: worker + name: worker +spec: + replicas: 2 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: worker + spec: + containers: + - args: + - ./worker + env: + - name: BITWARDENCLI_APPDATA_DIR + value: /BitwardenCLI + image: vwxyzjn/portwarden-base:1.6.0 + name: worker + resources: {} + stdin: true + tty: true + workingDir: /go/src/github.com/vwxyzjn/portwarden/web/worker + restartPolicy: Always +status: {}