diff --git a/connector/google/google.go b/connector/google/google.go
index eb9596cd..3f79a8a2 100644
--- a/connector/google/google.go
+++ b/connector/google/google.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-oidc/v3/oidc"
+	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	admin "google.golang.org/api/admin/directory/v1"
@@ -71,10 +72,17 @@ func (c *Config) Open(id string, logger log.Logger) (conn connector.Connector, e
 		scopes = append(scopes, "profile", "email")
 	}
 
-	srv, err := createDirectoryService(c.ServiceAccountFilePath, c.AdminEmail, logger)
-	if err != nil {
-		cancel()
-		return nil, fmt.Errorf("could not create directory service: %v", err)
+	var adminSrv *admin.Service
+
+	// Fixing a regression caused by default config fallback: https://github.com/dexidp/dex/issues/2699
+	if (c.ServiceAccountFilePath != "" && c.AdminEmail != "") || slices.Contains(scopes, "groups") {
+		srv, err := createDirectoryService(c.ServiceAccountFilePath, c.AdminEmail, logger)
+		if err != nil {
+			cancel()
+			return nil, fmt.Errorf("could not create directory service: %v", err)
+		}
+
+		adminSrv = srv
 	}
 
 	clientID := c.ClientID
@@ -97,7 +105,7 @@ func (c *Config) Open(id string, logger log.Logger) (conn connector.Connector, e
 		serviceAccountFilePath:         c.ServiceAccountFilePath,
 		adminEmail:                     c.AdminEmail,
 		fetchTransitiveGroupMembership: c.FetchTransitiveGroupMembership,
-		adminSrv:                       srv,
+		adminSrv:                       adminSrv,
 	}, nil
 }
 
diff --git a/go.mod b/go.mod
index 7df64216..539435cb 100644
--- a/go.mod
+++ b/go.mod
@@ -83,6 +83,7 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/text v0.3.7 // indirect
diff --git a/go.sum b/go.sum
index a3d41b73..a4c6d16e 100644
--- a/go.sum
+++ b/go.sum
@@ -440,6 +440,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 h1:fGZugkZk2UgYBxtpKmvub51Yno1LJDeEsRp2xGD+0gY=
+golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=