Linux Programming

How to List, Upload and Download files from an SFTP Server using golang

In this guide we are going to explore how to use Golang script to do operations on an SFTP server.

SFTP (Secure File Transfer Protocol) is a file transfer protocol that leverages a set of utilities that provide secure access to a remote computer to deliver secure communications. It relies on SSH.

Related Content

Prerequisites

To follow along:

  • Ensure you have golang installed locally.
  • Ensure you have access to an SFTP server - Username and password
  • Ensure you are familiar with the terminal

Creating the directory structure and initializing golang module

We need a directory that will have our content. Create it with this command:

1

mkdir gosftp

Switch to the directory and initialize a golang module:

1
2
3

$ cd gosftp
$ go mod init gosftp
go: creating new go.mod: module gosftp

This will create a file go.mod with this content:

1
2
3

module gosftp

go 1.22.6

Creating the script: Imports

Let’s not create the script. Create a file called main.go and add these imports:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

package main

import (
    "bufio"
    "fmt"
    "io"
    "log"
    "net"
    "net/url"
    "os"
    "path/filepath"
    "strings"
    "time"

    "golang.org/x/crypto/ssh"
    "golang.org/x/crypto/ssh/agent"

    "github.com/pkg/sftp"
)

Creating the script: Connecting to the server

Now that we have the imports, let’s use this code to initialize connection to the sftp server:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

    // Create a url
    rawurl := fmt.Sprintf("sftp://%v:%v@%v", sftpUser, sftpPass, sftpHost)

    // Parse the URL
    parsedUrl, err := url.Parse(rawurl)
    if err != nil {
        log.Fatalf("Failed to parse SFTP To Go URL: %s", err)
    }

    // Get user name and pass
    user := parsedUrl.User.Username()
    pass, _ := parsedUrl.User.Password()

    // Parse Host and Port
    host := parsedUrl.Host

    // Get hostkey
    hostKey := getHostKey(host)

    log.Printf("Connecting to %s ...\n", host)

    var auths []ssh.AuthMethod

    // Try to use $SSH_AUTH_SOCK which contains the path of the unix file socket that the sshd agent uses
    // for communication with other processes.
    if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
        auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers))
    }

    // Use password authentication if provided
    if pass != "" {
        auths = append(auths, ssh.Password(pass))
    }

    // Initialize client configuration
    config := ssh.ClientConfig{
        User: user,
        Auth: auths,
        // Auth: []ssh.AuthMethod{
        //  ssh.KeyboardInteractive(SshInteractive),
        // },

        // Uncomment to ignore host key check
        // HostKeyCallback: ssh.InsecureIgnoreHostKey(),
        HostKeyCallback: ssh.FixedHostKey(hostKey),
        // HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
        //  return nil
        // },
        Timeout: 30 * time.Second,
    }

    addr := fmt.Sprintf("%s:%s", host, sftpPort)

    // Connect to server
    conn, err := ssh.Dial("tcp", addr, &config)
    if err != nil {
        log.Fatalf("Failed to connec to host [%s]: %v", addr, err)
    }

    defer conn.Close()

    // Create new SFTP client
    sc, err := sftp.NewClient(conn)
    if err != nil {
        log.Fatalf("Unable to start SFTP subsystem: %v", err)
    }
    defer sc.Close()

Creating the script: Function to list files

Now let’s create a function to list files. We are using the connection to the sftp server to read contents of the remote dir then add to a list of of structs to return.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

func listFiles(sc sftp.Client, remoteDir string) (theFiles []remoteFiles, err error) {

    files, err := sc.ReadDir(remoteDir)
    if err != nil {
        return theFiles, fmt.Errorf("Unable to list remote dir: %v", err)
    }

    for _, f := range files {
        var name, modTime, size string

        name = f.Name()
        modTime = f.ModTime().Format("2006-01-02 15:04:05")
        size = fmt.Sprintf("%12d", f.Size())

        if f.IsDir() {
            name = name + "/"
            modTime = ""
            size = "PRE"
        }

        theFiles = append(theFiles, remoteFiles{
            Name:    name,
            Size:    size,
            ModTime: modTime,
        })
    }

    return theFiles, nil
}

Creating the script: Function to upload files

Let’s create a function to upload files to the sftp server. We will use the connection to open a file, then make remote directories recursively then copy the data from the local file

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

// Upload file to sftp server
func uploadFile(sc sftp.Client, localFile, remoteFile string) (err error) {
    log.Printf("Uploading [%s] to [%s] ...", localFile, remoteFile)

    srcFile, err := os.Open(localFile)
    if err != nil {
        return fmt.Errorf("Unable to open local file: %v", err)
    }
    defer srcFile.Close()

    // Make remote directories recursion
    parent := filepath.Dir(remoteFile)
    path := string(filepath.Separator)
    dirs := strings.Split(parent, path)
    for _, dir := range dirs {
        path = filepath.Join(path, dir)
        sc.Mkdir(path)
    }

    // Note: SFTP Go doesn't support O_RDWR mode
    dstFile, err := sc.OpenFile(remoteFile, (os.O_WRONLY | os.O_CREATE | os.O_TRUNC))
    if err != nil {
        return fmt.Errorf("Unable to open remote file: %v", err)
    }
    defer dstFile.Close()

    bytes, err := io.Copy(dstFile, srcFile)
    if err != nil {
        return fmt.Errorf("Unable to upload local file: %v", err)
    }
    log.Printf("%d bytes copied", bytes)

    return nil
}

Creating the script: Function to download files

This function downloads a file from the remote server given the remote path. In this function, we are creating a file in the tmp directory then copying the data from the remote file path.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

// Download file from sftp server
func downloadFile(sc sftp.Client, remoteFile, localFile string) (err error) {

    localPath := "/tmp/" + localFile

    log.Printf("Downloading [%s] to [%s] ...", remoteFile, localFile)
    // Note: SFTP To Go doesn't support O_RDWR mode
    srcFile, err := sc.OpenFile(remoteFile, (os.O_RDONLY))
    if err != nil {
        return fmt.Errorf("unable to open remote file: %v", err)
    }
    defer srcFile.Close()

    dstFile, err := os.Create(localPath)
    if err != nil {
        return fmt.Errorf("unable to open local file: %v", err)
    }
    defer dstFile.Close()

    bytes, err := io.Copy(dstFile, srcFile)
    if err != nil {
        return fmt.Errorf("unable to download remote file: %v", err)
    }
    log.Printf("%d bytes copied to %v", bytes, localPath)

    return nil
}

Creating the script: Full code

This is the full code for the script to do operations with SFTP using Golang:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267

package main

import (
    "bufio"
    "fmt"
    "io"
    "log"
    "net"
    "net/url"
    "os"
    "path/filepath"
    "strings"
    "time"

    "golang.org/x/crypto/ssh"
    "golang.org/x/crypto/ssh/agent"

    "github.com/pkg/sftp"
)

const (
    sftpUser = "citizix"
    sftpPass = "Str0ngP4ss"
    sftpHost = "10.2.11.10"
    sftpPort = "22"
)

func main() {
    // Create a url
    rawurl := fmt.Sprintf("sftp://%v:%v@%v", sftpUser, sftpPass, sftpHost)

    // Parse the URL
    parsedUrl, err := url.Parse(rawurl)
    if err != nil {
        log.Fatalf("Failed to parse SFTP To Go URL: %s", err)
    }

    // Get user name and pass
    user := parsedUrl.User.Username()
    pass, _ := parsedUrl.User.Password()

    // Parse Host and Port
    host := parsedUrl.Host

    // Get hostkey
    hostKey := getHostKey(host)

    log.Printf("Connecting to %s ...\n", host)

    var auths []ssh.AuthMethod

    // Try to use $SSH_AUTH_SOCK which contains the path of the unix file socket that the sshd agent uses
    // for communication with other processes.
    if aconn, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
        auths = append(auths, ssh.PublicKeysCallback(agent.NewClient(aconn).Signers))
    }

    // Use password authentication if provided
    if pass != "" {
        auths = append(auths, ssh.Password(pass))
    }

    // Initialize client configuration
    config := ssh.ClientConfig{
        User: user,
        Auth: auths,
        // Auth: []ssh.AuthMethod{
        //  ssh.KeyboardInteractive(SshInteractive),
        // },

        // Uncomment to ignore host key check
        // HostKeyCallback: ssh.InsecureIgnoreHostKey(),
        HostKeyCallback: ssh.FixedHostKey(hostKey),
        // HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
        //  return nil
        // },
        Timeout: 30 * time.Second,
    }

    addr := fmt.Sprintf("%s:%s", host, sftpPort)

    // Connect to server
    conn, err := ssh.Dial("tcp", addr, &config)
    if err != nil {
        log.Fatalf("Failed to connec to host [%s]: %v", addr, err)
    }

    defer conn.Close()

    // Create new SFTP client
    sc, err := sftp.NewClient(conn)
    if err != nil {
        log.Fatalf("Unable to start SFTP subsystem: %v", err)
    }
    defer sc.Close()

    // List files in the root directory .
    theFiles, err := listFiles(*sc, ".")
    if err != nil {
        log.Fatalf("failed to list files in .: %v", err)
    }

    log.Printf("Found Files in . Files")
    // Output each file name and size in bytes
    log.Printf("%19s %12s %s", "MOD TIME", "SIZE", "NAME")
    for _, theFile := range theFiles {
        log.Printf("%19s %12s %s", theFile.ModTime, theFile.Size, theFile.Name)
    }

    // Upload local file
    err = uploadFile(*sc, "/Users/etowett/Desktop/data.csv", "./citizix/data.csv")
    if err != nil {
        log.Fatalf("could not upload file: %v", err)
    }

    // Download remote file to local file.
    err = downloadFile(*sc, "citizix/data.csv", "data.csv")
    if err != nil {
        log.Fatalf("Could not download file data.csv; %v", err)
    }
    return
}

func SshInteractive(user, instruction string, questions []string, echos []bool) (answers []string, err error) {
    // Hack, check https://stackoverflow.com/questions/47102080/ssh-in-go-unable-to-authenticate-attempted-methods-none-no-supported-method
    answers = make([]string, len(questions))
    // The second parameter is unused
    for n, _ := range questions {
        answers[n] = sftpPass
    }

    return answers, nil
}

type remoteFiles struct {
    Name    string
    Size    string
    ModTime string
}

func listFiles(sc sftp.Client, remoteDir string) (theFiles []remoteFiles, err error) {

    files, err := sc.ReadDir(remoteDir)
    if err != nil {
        return theFiles, fmt.Errorf("Unable to list remote dir: %v", err)
    }

    for _, f := range files {
        var name, modTime, size string

        name = f.Name()
        modTime = f.ModTime().Format("2006-01-02 15:04:05")
        size = fmt.Sprintf("%12d", f.Size())

        if f.IsDir() {
            name = name + "/"
            modTime = ""
            size = "PRE"
        }

        theFiles = append(theFiles, remoteFiles{
            Name:    name,
            Size:    size,
            ModTime: modTime,
        })
    }

    return theFiles, nil
}

// Upload file to sftp server
func uploadFile(sc sftp.Client, localFile, remoteFile string) (err error) {
    log.Printf("Uploading [%s] to [%s] ...", localFile, remoteFile)

    srcFile, err := os.Open(localFile)
    if err != nil {
        return fmt.Errorf("Unable to open local file: %v", err)
    }
    defer srcFile.Close()

    // Make remote directories recursion
    parent := filepath.Dir(remoteFile)
    path := string(filepath.Separator)
    dirs := strings.Split(parent, path)
    for _, dir := range dirs {
        path = filepath.Join(path, dir)
        sc.Mkdir(path)
    }

    // Note: SFTP Go doesn't support O_RDWR mode
    dstFile, err := sc.OpenFile(remoteFile, (os.O_WRONLY | os.O_CREATE | os.O_TRUNC))
    if err != nil {
        return fmt.Errorf("Unable to open remote file: %v", err)
    }
    defer dstFile.Close()

    bytes, err := io.Copy(dstFile, srcFile)
    if err != nil {
        return fmt.Errorf("Unable to upload local file: %v", err)
    }
    log.Printf("%d bytes copied", bytes)

    return nil
}

// Download file from sftp server
func downloadFile(sc sftp.Client, remoteFile, localFile string) (err error) {

    log.Printf("Downloading [%s] to [%s] ...\n", remoteFile, localFile)
    // Note: SFTP To Go doesn't support O_RDWR mode
    srcFile, err := sc.OpenFile(remoteFile, (os.O_RDONLY))
    if err != nil {
        return fmt.Errorf("unable to open remote file: %v", err)
    }
    defer srcFile.Close()

    dstFile, err := os.Create(localFile)
    if err != nil {
        return fmt.Errorf("unable to open local file: %v", err)
    }
    defer dstFile.Close()

    bytes, err := io.Copy(dstFile, srcFile)
    if err != nil {
        return fmt.Errorf("unable to download remote file: %v", err)
    }
    log.Printf("%d bytes copied to %v", bytes, dstFile)

    return nil
}

// Get host key from local known hosts
func getHostKey(host string) ssh.PublicKey {
    // parse OpenSSH known_hosts file
    // ssh or use ssh-keyscan to get initial key
    file, err := os.Open(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts"))
    if err != nil {
        fmt.Fprintf(os.Stderr, "Unable to read known_hosts file: %v\n", err)
        os.Exit(1)
    }
    defer file.Close()

    scanner := bufio.NewScanner(file)
    var hostKey ssh.PublicKey
    for scanner.Scan() {
        fields := strings.Split(scanner.Text(), " ")
        if len(fields) != 3 {
            continue
        }
        if strings.Contains(fields[0], host) {
            var err error
            hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes())
            if err != nil {
                fmt.Fprintf(os.Stderr, "Error parsing %q: %v\n", fields[2], err)
                os.Exit(1)
            }
            break
        }
    }

    if hostKey == nil {
        fmt.Fprintf(os.Stderr, "No hostkey found for %s", host)
        os.Exit(1)
    }

    return hostKey
}

Building and testing the code

Now that we have our code, let’s build and test it.

First ensure all dependencies are downloaded using the go mod tidy command. This is my output:

1
2
3
4
5
6

$ go mod tidy

go: finding module for package golang.org/x/crypto/ssh
go: finding module for package github.com/pkg/sftp
go: found github.com/pkg/sftp in github.com/pkg/sftp v1.13.4
go: found golang.org/x/crypto/ssh in golang.org/x/crypto v0.0.0-20210921155107-089bfa567519

Next let’s build our code into a gosftp binary in the current directory:

1

go build -o gosftp

Now run the script. This is my output:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

$ ./gosftp

2021/10/08 13:10:36 Connecting to 10.2.11.10 ...
2021/10/08 13:10:43 Found Files in . Files
2021/10/08 13:10:43            MOD TIME         SIZE NAME
2021/10/08 13:10:43                              PRE etowett/
2021/10/08 13:10:43                              PRE citizix/
2021/10/08 13:10:43                              PRE PAYMENTDATA/
2021/10/08 13:10:43 Uploading [/Users/etowett/Desktop/data.csv] to [./citizix/data.csv] ...
2021/10/08 13:10:44 24 bytes copied
2021/10/08 13:10:45 Downloading [citizix/data.csv] to [data.csv] ...
2021/10/08 13:10:46 24 bytes copied to &{0xc000090a20}

Conclusion

In this article, we managed to create a script to list files in a remote sftp server, Upload files and download files.

comments powered by Disqus
© 2020 - 2024 Citizix, Inc.
Citizix Ltd
Built with Hugo
Theme Stack designed by Jimmy