Also move KubeStore and MemStore into their own package. RELNOTE: tsnet now supports providing a custom ipn.StateStore. Signed-off-by: Maisem Ali <maisem@tailscale.com>main
Maisem Ali
4 years ago
committed by
Maisem Ali
parent
d9a7205be5
commit
497324ddf6
@ -0,0 +1,85 @@ |
||||
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package kubestore contains an ipn.StateStore implementation using Kubernetes Secrets.
|
||||
|
||||
package kubestore |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"tailscale.com/ipn" |
||||
"tailscale.com/kube" |
||||
"tailscale.com/types/logger" |
||||
) |
||||
|
||||
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
|
||||
type Store struct { |
||||
client *kube.Client |
||||
secretName string |
||||
} |
||||
|
||||
// New returns a new Store that persists to the named secret.
|
||||
func New(_ logger.Logf, secretName string) (*Store, error) { |
||||
c, err := kube.New() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return &Store{ |
||||
client: c, |
||||
secretName: secretName, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *Store) String() string { return "kube.Store" } |
||||
|
||||
// ReadState implements the StateStore interface.
|
||||
func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) { |
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
||||
defer cancel() |
||||
|
||||
secret, err := s.client.GetSecret(ctx, s.secretName) |
||||
if err != nil { |
||||
if st, ok := err.(*kube.Status); ok && st.Code == 404 { |
||||
return nil, ipn.ErrStateNotExist |
||||
} |
||||
return nil, err |
||||
} |
||||
b, ok := secret.Data[string(id)] |
||||
if !ok { |
||||
return nil, ipn.ErrStateNotExist |
||||
} |
||||
return b, nil |
||||
} |
||||
|
||||
// WriteState implements the StateStore interface.
|
||||
func (s *Store) WriteState(id ipn.StateKey, bs []byte) error { |
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
||||
defer cancel() |
||||
|
||||
secret, err := s.client.GetSecret(ctx, s.secretName) |
||||
if err != nil { |
||||
if st, ok := err.(*kube.Status); ok && st.Code == 404 { |
||||
return s.client.CreateSecret(ctx, &kube.Secret{ |
||||
TypeMeta: kube.TypeMeta{ |
||||
APIVersion: "v1", |
||||
Kind: "Secret", |
||||
}, |
||||
ObjectMeta: kube.ObjectMeta{ |
||||
Name: s.secretName, |
||||
}, |
||||
Data: map[string][]byte{ |
||||
string(id): bs, |
||||
}, |
||||
}) |
||||
} |
||||
return err |
||||
} |
||||
secret.Data[string(id)] = bs |
||||
if err := s.client.UpdateSecret(ctx, secret); err != nil { |
||||
return err |
||||
} |
||||
return err |
||||
} |
||||
@ -0,0 +1,69 @@ |
||||
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package mem provides an in-memory ipn.StateStore implementation.
|
||||
package mem |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"sync" |
||||
|
||||
"tailscale.com/ipn" |
||||
"tailscale.com/types/logger" |
||||
) |
||||
|
||||
// New returns a new Store.
|
||||
func New(logger.Logf, string) (ipn.StateStore, error) { |
||||
return new(Store), nil |
||||
} |
||||
|
||||
// Store is an ipn.StateStore that keeps state in memory only.
|
||||
type Store struct { |
||||
mu sync.Mutex |
||||
cache map[ipn.StateKey][]byte |
||||
} |
||||
|
||||
func (s *Store) String() string { return "mem.Store" } |
||||
|
||||
// ReadState implements the StateStore interface.
|
||||
func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
bs, ok := s.cache[id] |
||||
if !ok { |
||||
return nil, ipn.ErrStateNotExist |
||||
} |
||||
return bs, nil |
||||
} |
||||
|
||||
// WriteState implements the StateStore interface.
|
||||
func (s *Store) WriteState(id ipn.StateKey, bs []byte) error { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
if s.cache == nil { |
||||
s.cache = map[ipn.StateKey][]byte{} |
||||
} |
||||
s.cache[id] = append([]byte(nil), bs...) |
||||
return nil |
||||
} |
||||
|
||||
// LoadFromJSON attempts to unmarshal json content into the
|
||||
// in-memory cache.
|
||||
func (s *Store) LoadFromJSON(data []byte) error { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
return json.Unmarshal(data, &s.cache) |
||||
} |
||||
|
||||
// ExportToJSON exports the content of the cache to
|
||||
// JSON formatted []byte.
|
||||
func (s *Store) ExportToJSON() ([]byte, error) { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
if len(s.cache) == 0 { |
||||
// Avoid "null" serialization.
|
||||
return []byte("{}"), nil |
||||
} |
||||
return json.MarshalIndent(s.cache, "", " ") |
||||
} |
||||
@ -0,0 +1,193 @@ |
||||
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package store provides various implementation of ipn.StateStore.
|
||||
package store |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"log" |
||||
"os" |
||||
"path/filepath" |
||||
"runtime" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"tailscale.com/atomicfile" |
||||
"tailscale.com/ipn" |
||||
"tailscale.com/ipn/store/mem" |
||||
"tailscale.com/paths" |
||||
"tailscale.com/types/logger" |
||||
) |
||||
|
||||
// Provider returns a StateStore for the provided path.
|
||||
// The arg is of the form "prefix:rest", where prefix was previously registered with Register.
|
||||
type Provider func(logf logger.Logf, arg string) (ipn.StateStore, error) |
||||
|
||||
var regOnce sync.Once |
||||
|
||||
var registerAvailableExternalStores func() |
||||
|
||||
func registerDefaultStores() { |
||||
Register("mem:", mem.New) |
||||
|
||||
if registerAvailableExternalStores != nil { |
||||
registerAvailableExternalStores() |
||||
} |
||||
} |
||||
|
||||
var knownStores map[string]Provider |
||||
|
||||
// New returns a StateStore based on the provided arg
|
||||
// and registered stores.
|
||||
// The arg is of the form "prefix:rest", where prefix was previously
|
||||
// registered with Register.
|
||||
//
|
||||
// By default the following stores are registered:
|
||||
//
|
||||
// * if the string begins with "mem:", the suffix
|
||||
// is ignored and an in-memory store is used.
|
||||
// * (Linux-only) if the string begins with "arn:",
|
||||
// the suffix an AWS ARN for an SSM.
|
||||
// * (Linux-only) if the string begins with "kube:",
|
||||
// the suffix is a Kubernetes secret name
|
||||
// * In all other cases, the path is treated as a filepath.
|
||||
func New(logf logger.Logf, path string) (ipn.StateStore, error) { |
||||
regOnce.Do(registerDefaultStores) |
||||
for prefix, sf := range knownStores { |
||||
if strings.HasPrefix(path, prefix) { |
||||
// We can't strip the prefix here as some NewStoreFunc (like arn:)
|
||||
// expect the prefix.
|
||||
return sf(logf, path) |
||||
} |
||||
} |
||||
if runtime.GOOS == "windows" { |
||||
path = TryWindowsAppDataMigration(logf, path) |
||||
} |
||||
return NewFileStore(logf, path) |
||||
} |
||||
|
||||
// Register registers a prefix to be used for
|
||||
// NewStore. It panics if the prefix is empty, or if the
|
||||
// prefix is already registered.
|
||||
// The provided fn is called with the path passed to NewStore;
|
||||
// the prefix is not stripped.
|
||||
func Register(prefix string, fn Provider) { |
||||
if len(prefix) == 0 { |
||||
panic("prefix is empty") |
||||
} |
||||
if _, ok := knownStores[prefix]; ok { |
||||
panic(fmt.Sprintf("%q already registered", prefix)) |
||||
} |
||||
if knownStores == nil { |
||||
knownStores = make(map[string]Provider) |
||||
} |
||||
knownStores[prefix] = fn |
||||
} |
||||
|
||||
// TryWindowsAppDataMigration attempts to copy the Windows state file
|
||||
// from its old location to the new location. (Issue 2856)
|
||||
//
|
||||
// Tailscale 1.14 and before stored state under %LocalAppData%
|
||||
// (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local"
|
||||
// when tailscaled.exe is running as a non-user system service).
|
||||
// However it is frequently cleared for almost any reason: Windows
|
||||
// updates, System Restore, even various System Cleaner utilities.
|
||||
//
|
||||
// Returns a string of the path to use for the state file.
|
||||
// This will be a fallback %LocalAppData% path if migration fails,
|
||||
// a %ProgramData% path otherwise.
|
||||
func TryWindowsAppDataMigration(logf logger.Logf, path string) string { |
||||
if path != paths.DefaultTailscaledStateFile() { |
||||
// If they're specifying a non-default path, just trust that they know
|
||||
// what they are doing.
|
||||
return path |
||||
} |
||||
oldFile := paths.LegacyStateFilePath() |
||||
return paths.TryConfigFileMigration(logf, oldFile, path) |
||||
} |
||||
|
||||
// FileStore is a StateStore that uses a JSON file for persistence.
|
||||
type FileStore struct { |
||||
path string |
||||
|
||||
mu sync.RWMutex |
||||
cache map[ipn.StateKey][]byte |
||||
} |
||||
|
||||
// Path returns the path that NewFileStore was called with.
|
||||
func (s *FileStore) Path() string { return s.path } |
||||
|
||||
func (s *FileStore) String() string { return fmt.Sprintf("FileStore(%q)", s.path) } |
||||
|
||||
// NewFileStore returns a new file store that persists to path.
|
||||
func NewFileStore(_ logger.Logf, path string) (ipn.StateStore, error) { |
||||
// We unconditionally call this to ensure that our perms are correct
|
||||
if err := paths.MkStateDir(filepath.Dir(path)); err != nil { |
||||
return nil, fmt.Errorf("creating state directory: %w", err) |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(path) |
||||
|
||||
// Treat an empty file as a missing file.
|
||||
// (https://github.com/tailscale/tailscale/issues/895#issuecomment-723255589)
|
||||
if err == nil && len(bs) == 0 { |
||||
log.Printf("ipn.NewFileStore(%q): file empty; treating it like a missing file [warning]", path) |
||||
err = os.ErrNotExist |
||||
} |
||||
|
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
// Write out an initial file, to verify that we can write
|
||||
// to the path.
|
||||
if err = atomicfile.WriteFile(path, []byte("{}"), 0600); err != nil { |
||||
return nil, err |
||||
} |
||||
return &FileStore{ |
||||
path: path, |
||||
cache: map[ipn.StateKey][]byte{}, |
||||
}, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
ret := &FileStore{ |
||||
path: path, |
||||
cache: map[ipn.StateKey][]byte{}, |
||||
} |
||||
if err := json.Unmarshal(bs, &ret.cache); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return ret, nil |
||||
} |
||||
|
||||
// ReadState implements the StateStore interface.
|
||||
func (s *FileStore) ReadState(id ipn.StateKey) ([]byte, error) { |
||||
s.mu.RLock() |
||||
defer s.mu.RUnlock() |
||||
bs, ok := s.cache[id] |
||||
if !ok { |
||||
return nil, ipn.ErrStateNotExist |
||||
} |
||||
return bs, nil |
||||
} |
||||
|
||||
// WriteState implements the StateStore interface.
|
||||
func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
if bytes.Equal(s.cache[id], bs) { |
||||
return nil |
||||
} |
||||
s.cache[id] = append([]byte(nil), bs...) |
||||
bs, err := json.MarshalIndent(s.cache, "", " ") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return atomicfile.WriteFile(s.path, bs, 0600) |
||||
} |
||||
@ -0,0 +1,26 @@ |
||||
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package store |
||||
|
||||
import ( |
||||
"strings" |
||||
|
||||
"tailscale.com/ipn" |
||||
"tailscale.com/ipn/store/awsstore" |
||||
"tailscale.com/ipn/store/kubestore" |
||||
"tailscale.com/types/logger" |
||||
) |
||||
|
||||
func init() { |
||||
registerAvailableExternalStores = registerExternalStores |
||||
} |
||||
|
||||
func registerExternalStores() { |
||||
Register("kube:", func(logf logger.Logf, path string) (ipn.StateStore, error) { |
||||
secretName := strings.TrimPrefix(path, "kube:") |
||||
return kubestore.New(logf, secretName) |
||||
}) |
||||
Register("arn:", awsstore.New) |
||||
} |
||||
@ -0,0 +1,191 @@ |
||||
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package store |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"tailscale.com/ipn" |
||||
"tailscale.com/ipn/store/mem" |
||||
"tailscale.com/tstest" |
||||
"tailscale.com/types/logger" |
||||
) |
||||
|
||||
func TestNewStore(t *testing.T) { |
||||
regOnce.Do(registerDefaultStores) |
||||
t.Cleanup(func() { |
||||
knownStores = map[string]Provider{} |
||||
registerDefaultStores() |
||||
}) |
||||
knownStores = map[string]Provider{} |
||||
|
||||
type store1 struct { |
||||
ipn.StateStore |
||||
path string |
||||
} |
||||
|
||||
type store2 struct { |
||||
ipn.StateStore |
||||
path string |
||||
} |
||||
|
||||
Register("arn:", func(_ logger.Logf, path string) (ipn.StateStore, error) { |
||||
return &store1{new(mem.Store), path}, nil |
||||
}) |
||||
Register("kube:", func(_ logger.Logf, path string) (ipn.StateStore, error) { |
||||
return &store2{new(mem.Store), path}, nil |
||||
}) |
||||
Register("mem:", func(_ logger.Logf, path string) (ipn.StateStore, error) { |
||||
return new(mem.Store), nil |
||||
}) |
||||
|
||||
path := "mem:abcd" |
||||
if s, err := New(t.Logf, path); err != nil { |
||||
t.Fatalf("%q: %v", path, err) |
||||
} else if _, ok := s.(*mem.Store); !ok { |
||||
t.Fatalf("%q: got: %T, want: %T", path, s, new(mem.Store)) |
||||
} |
||||
|
||||
path = "arn:foo" |
||||
if s, err := New(t.Logf, path); err != nil { |
||||
t.Fatalf("%q: %v", path, err) |
||||
} else if _, ok := s.(*store1); !ok { |
||||
t.Fatalf("%q: got: %T, want: %T", path, s, new(store1)) |
||||
} |
||||
|
||||
path = "kube:abcd" |
||||
if s, err := New(t.Logf, path); err != nil { |
||||
t.Fatalf("%q: %v", path, err) |
||||
} else if _, ok := s.(*store2); !ok { |
||||
t.Fatalf("%q: got: %T, want: %T", path, s, new(store2)) |
||||
} |
||||
f, err := os.CreateTemp("", "") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := f.Close(); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
t.Cleanup(func() { |
||||
os.Remove(f.Name()) |
||||
}) |
||||
|
||||
path = f.Name() |
||||
if s, err := New(t.Logf, path); err != nil { |
||||
t.Fatalf("%q: %v", path, err) |
||||
} else if _, ok := s.(*FileStore); !ok { |
||||
t.Fatalf("%q: got: %T, want: %T", path, s, new(FileStore)) |
||||
} |
||||
} |
||||
|
||||
func testStoreSemantics(t *testing.T, store ipn.StateStore) { |
||||
t.Helper() |
||||
|
||||
tests := []struct { |
||||
// if true, data is data to write. If false, data is expected
|
||||
// output of read.
|
||||
write bool |
||||
id ipn.StateKey |
||||
data string |
||||
// If write=false, true if we expect a not-exist error.
|
||||
notExists bool |
||||
}{ |
||||
{ |
||||
id: "foo", |
||||
notExists: true, |
||||
}, |
||||
{ |
||||
write: true, |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "baz", |
||||
notExists: true, |
||||
}, |
||||
{ |
||||
write: true, |
||||
id: "baz", |
||||
data: "quux", |
||||
}, |
||||
{ |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "baz", |
||||
data: "quux", |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
if test.write { |
||||
if err := store.WriteState(test.id, []byte(test.data)); err != nil { |
||||
t.Errorf("writing %q to %q: %v", test.data, test.id, err) |
||||
} |
||||
} else { |
||||
bs, err := store.ReadState(test.id) |
||||
if err != nil { |
||||
if test.notExists && err == ipn.ErrStateNotExist { |
||||
continue |
||||
} |
||||
t.Errorf("reading %q: %v", test.id, err) |
||||
continue |
||||
} |
||||
if string(bs) != test.data { |
||||
t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestMemoryStore(t *testing.T) { |
||||
tstest.PanicOnLog() |
||||
|
||||
store := new(mem.Store) |
||||
testStoreSemantics(t, store) |
||||
} |
||||
|
||||
func TestFileStore(t *testing.T) { |
||||
tstest.PanicOnLog() |
||||
|
||||
dir := t.TempDir() |
||||
path := filepath.Join(dir, "test-file-store.conf") |
||||
|
||||
store, err := NewFileStore(nil, path) |
||||
if err != nil { |
||||
t.Fatalf("creating file store failed: %v", err) |
||||
} |
||||
|
||||
testStoreSemantics(t, store) |
||||
|
||||
// Build a brand new file store and check that both IDs written
|
||||
// above are still there.
|
||||
store, err = NewFileStore(nil, path) |
||||
if err != nil { |
||||
t.Fatalf("creating second file store failed: %v", err) |
||||
} |
||||
|
||||
expected := map[ipn.StateKey]string{ |
||||
"foo": "bar", |
||||
"baz": "quux", |
||||
} |
||||
for key, want := range expected { |
||||
bs, err := store.ReadState(key) |
||||
if err != nil { |
||||
t.Errorf("reading %q (2nd store): %v", key, err) |
||||
continue |
||||
} |
||||
if string(bs) != want { |
||||
t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want) |
||||
} |
||||
} |
||||
} |
||||
@ -1,120 +0,0 @@ |
||||
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package ipn |
||||
|
||||
import ( |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"tailscale.com/tstest" |
||||
) |
||||
|
||||
func testStoreSemantics(t *testing.T, store StateStore) { |
||||
t.Helper() |
||||
|
||||
tests := []struct { |
||||
// if true, data is data to write. If false, data is expected
|
||||
// output of read.
|
||||
write bool |
||||
id StateKey |
||||
data string |
||||
// If write=false, true if we expect a not-exist error.
|
||||
notExists bool |
||||
}{ |
||||
{ |
||||
id: "foo", |
||||
notExists: true, |
||||
}, |
||||
{ |
||||
write: true, |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "baz", |
||||
notExists: true, |
||||
}, |
||||
{ |
||||
write: true, |
||||
id: "baz", |
||||
data: "quux", |
||||
}, |
||||
{ |
||||
id: "foo", |
||||
data: "bar", |
||||
}, |
||||
{ |
||||
id: "baz", |
||||
data: "quux", |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
if test.write { |
||||
if err := store.WriteState(test.id, []byte(test.data)); err != nil { |
||||
t.Errorf("writing %q to %q: %v", test.data, test.id, err) |
||||
} |
||||
} else { |
||||
bs, err := store.ReadState(test.id) |
||||
if err != nil { |
||||
if test.notExists && err == ErrStateNotExist { |
||||
continue |
||||
} |
||||
t.Errorf("reading %q: %v", test.id, err) |
||||
continue |
||||
} |
||||
if string(bs) != test.data { |
||||
t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestMemoryStore(t *testing.T) { |
||||
tstest.PanicOnLog() |
||||
|
||||
store := &MemoryStore{} |
||||
testStoreSemantics(t, store) |
||||
} |
||||
|
||||
func TestFileStore(t *testing.T) { |
||||
tstest.PanicOnLog() |
||||
|
||||
dir := t.TempDir() |
||||
path := filepath.Join(dir, "test-file-store.conf") |
||||
|
||||
store, err := NewFileStore(path) |
||||
if err != nil { |
||||
t.Fatalf("creating file store failed: %v", err) |
||||
} |
||||
|
||||
testStoreSemantics(t, store) |
||||
|
||||
// Build a brand new file store and check that both IDs written
|
||||
// above are still there.
|
||||
store, err = NewFileStore(path) |
||||
if err != nil { |
||||
t.Fatalf("creating second file store failed: %v", err) |
||||
} |
||||
|
||||
expected := map[StateKey]string{ |
||||
"foo": "bar", |
||||
"baz": "quux", |
||||
} |
||||
for key, want := range expected { |
||||
bs, err := store.ReadState(key) |
||||
if err != nil { |
||||
t.Errorf("reading %q (2nd store): %v", key, err) |
||||
continue |
||||
} |
||||
if string(bs) != want { |
||||
t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want) |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue