diff --git a/connector/google/google.go b/connector/google/google.go
index 313858f6..eb9596cd 100644
--- a/connector/google/google.go
+++ b/connector/google/google.go
@@ -71,13 +71,10 @@ func (c *Config) Open(id string, logger log.Logger) (conn connector.Connector, e
 		scopes = append(scopes, "profile", "email")
 	}
 
-	var srv *admin.Service
-	if len(c.Groups) > 0 {
-		srv, err = createDirectoryService(c.ServiceAccountFilePath, c.AdminEmail, logger)
-		if err != nil {
-			cancel()
-			return nil, fmt.Errorf("could not create directory service: %v", err)
-		}
+	srv, err := createDirectoryService(c.ServiceAccountFilePath, c.AdminEmail, logger)
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("could not create directory service: %v", err)
 	}
 
 	clientID := c.ClientID
@@ -286,7 +283,9 @@ func (c *googleConnector) getGroups(email string, fetchTransitiveGroupMembership
 // the google admin api. If no serviceAccountFilePath is defined, the application default credential
 // is used.
 func createDirectoryService(serviceAccountFilePath, email string, logger log.Logger) (*admin.Service, error) {
-	if email == "" {
+	// We know impersonation is required when using a service account credential
+	// TODO: or is it?
+	if email == "" && serviceAccountFilePath != "" {
 		return nil, fmt.Errorf("directory service requires adminEmail")
 	}
 
@@ -311,7 +310,12 @@ func createDirectoryService(serviceAccountFilePath, email string, logger log.Log
 	if err != nil {
 		return nil, fmt.Errorf("unable to parse credentials to config: %v", err)
 	}
-	config.Subject = email
+
+	// Only attempt impersonation when there is a user configured
+	if email != "" {
+		config.Subject = email
+	}
+
 	return admin.NewService(ctx, option.WithHTTPClient(config.Client(ctx)))
 }
 
diff --git a/connector/google/google_test.go b/connector/google/google_test.go
index 26241ab6..b0c4f3a2 100644
--- a/connector/google/google_test.go
+++ b/connector/google/google_test.go
@@ -72,22 +72,13 @@ func TestOpen(t *testing.T) {
 	assert.Nil(t, err)
 
 	for name, reference := range map[string]testCase{
-		"not_requesting_groups": {
-			config: &Config{
-				ClientID:     "testClient",
-				ClientSecret: "testSecret",
-				RedirectURI:  ts.URL + "/callback",
-				Scopes:       []string{"openid"},
-			},
-			expectedErr: "",
-		},
 		"missing_admin_email": {
 			config: &Config{
-				ClientID:     "testClient",
-				ClientSecret: "testSecret",
-				RedirectURI:  ts.URL + "/callback",
-				Scopes:       []string{"openid", "groups"},
-				Groups:       []string{"someGroup"},
+				ClientID:               "testClient",
+				ClientSecret:           "testSecret",
+				RedirectURI:            ts.URL + "/callback",
+				Scopes:                 []string{"openid", "groups"},
+				ServiceAccountFilePath: serviceAccountFilePath,
 			},
 			expectedErr: "requires adminEmail",
 		},
@@ -99,7 +90,6 @@ func TestOpen(t *testing.T) {
 				Scopes:                 []string{"openid", "groups"},
 				AdminEmail:             "foo@bar.com",
 				ServiceAccountFilePath: "not_found.json",
-				Groups:                 []string{"someGroup"},
 			},
 			expectedErr: "error reading credentials",
 		},
@@ -111,7 +101,6 @@ func TestOpen(t *testing.T) {
 				Scopes:                 []string{"openid", "groups"},
 				AdminEmail:             "foo@bar.com",
 				ServiceAccountFilePath: serviceAccountFilePath,
-				Groups:                 []string{"someGroup"},
 			},
 			expectedErr: "",
 		},
@@ -122,7 +111,6 @@ func TestOpen(t *testing.T) {
 				RedirectURI:  ts.URL + "/callback",
 				Scopes:       []string{"openid", "groups"},
 				AdminEmail:   "foo@bar.com",
-				Groups:       []string{"someGroup"},
 			},
 			adc:         serviceAccountFilePath,
 			expectedErr: "",
@@ -135,7 +123,6 @@ func TestOpen(t *testing.T) {
 				Scopes:                 []string{"openid", "groups"},
 				AdminEmail:             "foo@bar.com",
 				ServiceAccountFilePath: serviceAccountFilePath,
-				Groups:                 []string{"someGroup"},
 			},
 			adc:         "/dev/null",
 			expectedErr: "",