Merge pull request #293 from loki-project/onion-requests

Binary Onion Routing Protocol & More Granular Path Error Handling
pull/301/head
Niels Andriesse 5 years ago committed by GitHub
commit 0348289561
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1 +1 @@
Subproject commit 91063358ff177892d4809803a226e04db350d9ae
Subproject commit 11d995a37fa82d8a3592e98d1d16bdf81dc21e7c

@ -17,10 +17,7 @@ final class PathStatusView : UIView {
layer.cornerRadius = Values.pathStatusViewSize / 2
layer.masksToBounds = false
if OnionRequestAPI.paths.count < OnionRequestAPI.pathCount {
let storage = OWSPrimaryStorage.shared()
storage.dbReadConnection.read { transaction in
OnionRequestAPI.paths = storage.getOnionRequestPaths(in: transaction)
}
OnionRequestAPI.paths = Storage.getOnionRequestPaths()
}
let color = (OnionRequestAPI.paths.count >= OnionRequestAPI.pathCount) ? Colors.accent : Colors.pathsBuilding
setColor(to: color, isAnimated: false)

@ -52,10 +52,7 @@ final class IP2Country {
func populateCacheIfNeeded() -> Bool {
if OnionRequestAPI.paths.count < OnionRequestAPI.pathCount {
let storage = OWSPrimaryStorage.shared()
storage.dbReadConnection.read { transaction in
OnionRequestAPI.paths = storage.getOnionRequestPaths(in: transaction)
}
OnionRequestAPI.paths = Storage.getOnionRequestPaths()
}
let paths = OnionRequestAPI.paths
guard paths.count >= OnionRequestAPI.pathCount else { return false }

@ -7,8 +7,8 @@ final class RestoreVC : BaseVC {
private var bottomConstraint: NSLayoutConstraint!
// MARK: Components
private lazy var mnemonicTextField: TextField = {
let result = TextField(placeholder: NSLocalizedString("vc_restore_seed_text_field_hint", comment: ""))
private lazy var mnemonicTextView: TextView = {
let result = TextView(placeholder: NSLocalizedString("vc_restore_seed_text_field_hint", comment: ""))
result.layer.borderColor = Colors.text.cgColor
return result
}()
@ -77,7 +77,7 @@ final class RestoreVC : BaseVC {
restoreButtonContainer.pin(.trailing, to: .trailing, of: restoreButton, withInset: Values.massiveSpacing)
restoreButtonContainer.pin(.bottom, to: .bottom, of: restoreButton)
// Set up top stack view
let topStackView = UIStackView(arrangedSubviews: [ titleLabel, spacer1, explanationLabel, spacer2, mnemonicTextField, spacer3, legalLabel ])
let topStackView = UIStackView(arrangedSubviews: [ titleLabel, spacer1, explanationLabel, spacer2, mnemonicTextView, spacer3, legalLabel ])
topStackView.axis = .vertical
topStackView.alignment = .fill
// Set up top stack view container
@ -111,7 +111,7 @@ final class RestoreVC : BaseVC {
// On small screens we hide the legal label when the keyboard is up, but it's important that the user sees it so
// in those instances we don't make the keyboard come up automatically
if !isIPhone5OrSmaller {
mnemonicTextField.becomeFirstResponder()
mnemonicTextView.becomeFirstResponder()
}
}
@ -121,7 +121,7 @@ final class RestoreVC : BaseVC {
// MARK: General
@objc private func dismissKeyboard() {
mnemonicTextField.resignFirstResponder()
mnemonicTextView.resignFirstResponder()
}
// MARK: Updating
@ -159,7 +159,7 @@ final class RestoreVC : BaseVC {
alert.addAction(UIAlertAction(title: NSLocalizedString("OK", comment: ""), style: .default, handler: nil))
presentAlert(alert)
}
let mnemonic = mnemonicTextField.text!
let mnemonic = mnemonicTextView.text!
do {
let hexEncodedSeed = try Mnemonic.decode(mnemonic: mnemonic)
let seed = Data(hex: hexEncodedSeed)
@ -171,7 +171,7 @@ final class RestoreVC : BaseVC {
TSAccountManager.sharedInstance().phoneNumberAwaitingVerification = keyPair.hexEncodedPublicKey
OWSPrimaryStorage.shared().setRestorationTime(Date().timeIntervalSince1970)
UserDefaults.standard[.hasViewedSeed] = true
mnemonicTextField.resignFirstResponder()
mnemonicTextView.resignFirstResponder()
Timer.scheduledTimer(withTimeInterval: 0.25, repeats: false) { _ in
let displayNameVC = DisplayNameVC()
self.navigationController!.pushViewController(displayNameVC, animated: true)

@ -64,8 +64,8 @@ public final class Values : NSObject {
@objc public static let onboardingButtonBottomOffset = isIPhone5OrSmaller ? CGFloat(52) : CGFloat(72)
// MARK: - Animation Values
@objc public static let fakeChatStartDelay: TimeInterval = 1.5
@objc public static let fakeChatStartDelay: TimeInterval = 1
@objc public static let fakeChatAnimationDuration: TimeInterval = 0.4
@objc public static let fakeChatDelay: TimeInterval = 2
@objc public static let fakeChatDelay: TimeInterval = 1.5
@objc public static let fakeChatMessagePopAnimationStartScale: CGFloat = 0.6
}

@ -11,11 +11,12 @@ public final class FileServerAPI : DotNetAPI {
public static let maxFileSize = 10_000_000 // 10 MB
/// The file server has a file size limit of `maxFileSize`, which the Service Nodes try to enforce as well. However, the limit applied by the Service Nodes
/// is on the **HTTP request** and not the file size. Because of onion request encryption, a file that's about 4 MB will result in a request that's about 18 MB.
/// On average the multiplier appears to be about 6, so when checking whether the file will exceed the file size limit when uploading a file we just divide
/// the size of the file by this number. The alternative would be to actually check the size of the HTTP request but that's only possible after proof of work
/// has been calculated and the onion request encryption has happened, which takes several seconds.
public static let fileSizeORMultiplier: Double = 6
/// is on the **HTTP request** and not the actual file size. Because the file server expects the file data to be base 64 encoded, the size of the HTTP
/// request for a given file will be at least `ceil(n / 3) * 4` bytes, where n is the file size in bytes. This is the minimum size because there might also
/// be other parameters in the request. On average the multiplier appears to be about 1.5, so when checking whether the file will exceed the file size limit when
/// uploading a file we just divide the size of the file by this number. The alternative would be to actually check the size of the HTTP request but that's only
/// possible after proof of work has been calculated and the onion request encryption has happened, which takes several seconds.
public static let fileSizeORMultiplier: Double = 1.5
@objc public static let server = "https://file.getsession.org"
@objc public static let fileStorageBucketURL = "https://file-static.lokinet.org"

@ -3,6 +3,15 @@ import PromiseKit
extension OnionRequestAPI {
internal static func encode(ciphertext: Data, json: JSON) throws -> Data {
// The encoding of V2 onion requests looks like: | 4 bytes: size N of ciphertext | N bytes: ciphertext | json as utf8 |
guard JSONSerialization.isValidJSONObject(json) else { throw HTTP.Error.invalidJSON }
let jsonAsData = try JSONSerialization.data(withJSONObject: json, options: [ .fragmentsAllowed ])
let ciphertextSize = Int32(ciphertext.count).littleEndian
let ciphertextSizeAsData = withUnsafePointer(to: ciphertextSize) { Data(bytes: $0, count: MemoryLayout<Int32>.size) }
return ciphertextSizeAsData + ciphertext + jsonAsData
}
/// Encrypts `payload` for `destination` and returns the result. Use this to build the core of an onion request.
internal static func encrypt(_ payload: JSON, for destination: Destination) -> Promise<EncryptionResult> {
let (promise, seal) = Promise<EncryptionResult>.pending()
@ -14,10 +23,7 @@ extension OnionRequestAPI {
case .snode(let snode):
guard let snodeX25519PublicKey = snode.publicKeySet?.x25519Key else { return seal.reject(Error.snodePublicKeySetMissing) }
let payloadAsData = try JSONSerialization.data(withJSONObject: payload, options: [ .fragmentsAllowed ])
let payloadAsString = String(data: payloadAsData, encoding: .utf8)! // Snodes only accept this as a string
let wrapper: JSON = [ "body" : payloadAsString, "headers" : "" ]
guard JSONSerialization.isValidJSONObject(wrapper) else { return seal.reject(HTTP.Error.invalidJSON) }
let plaintext = try JSONSerialization.data(withJSONObject: wrapper, options: [ .fragmentsAllowed ])
let plaintext = try encode(ciphertext: payloadAsData, json: [ "headers" : "" ])
let result = try EncryptionUtilities.encrypt(plaintext, using: snodeX25519PublicKey)
seal.fulfill(result)
case .server(_, let serverX25519PublicKey):
@ -42,9 +48,8 @@ extension OnionRequestAPI {
guard let snodeED25519PublicKey = snode.publicKeySet?.ed25519Key else { return seal.reject(Error.snodePublicKeySetMissing) }
parameters = [ "destination" : snodeED25519PublicKey ]
case .server(let host, _):
parameters = [ "host" : host, "target" : "/loki/v1/lsrpc", "method" : "POST" ]
parameters = [ "host" : host, "target" : "/loki/v2/lsrpc", "method" : "POST" ]
}
parameters["ciphertext"] = previousEncryptionResult.ciphertext.base64EncodedString()
parameters["ephemeral_key"] = previousEncryptionResult.ephemeralPublicKey.toHexString()
let x25519PublicKey: String
switch lhs {
@ -55,8 +60,7 @@ extension OnionRequestAPI {
x25519PublicKey = serverX25519PublicKey
}
do {
guard JSONSerialization.isValidJSONObject(parameters) else { return seal.reject(HTTP.Error.invalidJSON) }
let plaintext = try JSONSerialization.data(withJSONObject: parameters, options: [ .fragmentsAllowed ])
let plaintext = try encode(ciphertext: previousEncryptionResult.ciphertext, json: parameters)
let result = try EncryptionUtilities.encrypt(plaintext, using: x25519PublicKey)
seal.fulfill(result)
} catch (let error) {

@ -56,7 +56,7 @@ public enum OnionRequestAPI {
let timeout: TimeInterval = 3 // Use a shorter timeout for testing
HTTP.execute(.get, url, timeout: timeout).done2 { rawResponse in
guard let json = rawResponse as? JSON, let version = json["version"] as? String else { return seal.reject(Error.missingSnodeVersion) }
if version >= "2.0.0" {
if version >= "2.0.7" {
seal.fulfill(())
} else {
print("[Loki] [Onion Request API] Unsupported snode version: \(version).")
@ -126,7 +126,7 @@ public enum OnionRequestAPI {
OnionRequestAPI.paths = paths
try! Storage.writeSync { transaction in
print("[Loki] Persisting onion request paths to database.")
OWSPrimaryStorage.shared().setOnionRequestPaths(paths, in: transaction)
Storage.setOnionRequestPaths(paths, using: transaction)
}
DispatchQueue.main.async {
NotificationCenter.default.post(name: .pathsBuilt, object: nil)
@ -137,19 +137,14 @@ public enum OnionRequestAPI {
}
/// Returns a `Path` to be used for building an onion request. Builds new paths as needed.
///
/// - Note: Exposed for testing purposes.
private static func getPath(excluding snode: Snode?) -> Promise<Path> {
guard pathSize >= 1 else { preconditionFailure("Can't build path of size zero.") }
var paths = OnionRequestAPI.paths
if paths.count < pathCount {
let storage = OWSPrimaryStorage.shared()
storage.dbReadConnection.read { transaction in
paths = storage.getOnionRequestPaths(in: transaction)
OnionRequestAPI.paths = paths
if paths.count >= pathCount {
guardSnodes.formUnion([ paths[0][0], paths[1][0] ])
}
paths = Storage.getOnionRequestPaths()
OnionRequestAPI.paths = paths
if paths.count >= pathCount {
guardSnodes.formUnion([ paths[0][0], paths[1][0] ])
}
}
// randomElement() uses the system's default random generator, which is cryptographically secure
@ -172,15 +167,35 @@ public enum OnionRequestAPI {
}
}
private static func dropAllPaths() {
paths.removeAll()
private static func dropGuardSnode(_ snode: Snode) {
guardSnodes = guardSnodes.filter { $0 != snode }
}
private static func drop(_ snode: Snode) throws {
var oldPaths = paths
guard let pathIndex = oldPaths.firstIndex(where: { $0.contains(snode) }) else { return }
var path = oldPaths[pathIndex]
guard let snodeIndex = path.firstIndex(of: snode) else { return }
path.remove(at: snodeIndex)
let unusedSnodes = SnodeAPI.snodePool.subtracting(oldPaths.flatMap { $0 })
guard !unusedSnodes.isEmpty else { throw Error.insufficientSnodes }
// randomElement() uses the system's default random generator, which is cryptographically secure
path.append(unusedSnodes.randomElement()!)
// Don't test the new snode as this would reveal the user's IP
oldPaths.remove(at: pathIndex)
let newPaths = oldPaths + [ path ]
paths = newPaths
try! Storage.writeSync { transaction in
OWSPrimaryStorage.shared().clearOnionRequestPaths(in: transaction)
print("[Loki] Persisting onion request paths to database.")
Storage.setOnionRequestPaths(newPaths, using: transaction)
}
}
private static func dropGuardSnode(_ snode: Snode) {
guardSnodes = guardSnodes.filter { $0 != snode }
private static func dropAllPaths() {
paths.removeAll()
try! Storage.writeSync { transaction in
Storage.clearOnionRequestPaths(using: transaction)
}
}
/// Builds an onion around `payload` and returns the result.
@ -265,8 +280,8 @@ public enum OnionRequestAPI {
}
let payload: JSON = [
"body" : parametersAsString,
"endpoint": endpoint,
"method" : request.httpMethod,
"endpoint" : endpoint,
"method" : request.httpMethod!,
"headers" : headers
]
let destination = Destination.server(host: host, x25519PublicKey: x25519PublicKey)
@ -282,18 +297,23 @@ public enum OnionRequestAPI {
SnodeAPI.workQueue.async { // Avoid race conditions on `guardSnodes` and `paths`
buildOnion(around: payload, targetedAt: destination).done2 { intermediate in
guardSnode = intermediate.guardSnode
let url = "\(guardSnode.address):\(guardSnode.port)/onion_req"
let url = "\(guardSnode.address):\(guardSnode.port)/onion_req/v2"
let finalEncryptionResult = intermediate.finalEncryptionResult
let onion = finalEncryptionResult.ciphertext
if case Destination.server = destination, Double(onion.count) > 0.75 * Double(FileServerAPI.maxFileSize) {
print("[Loki] Approaching request size limit: ~\(onion.count) bytes.")
}
let parameters: JSON = [
"ciphertext" : onion.base64EncodedString(),
"ephemeral_key" : finalEncryptionResult.ephemeralPublicKey.toHexString()
]
let body: Data
do {
body = try encode(ciphertext: onion, json: parameters)
} catch {
return seal.reject(error)
}
let destinationSymmetricKey = intermediate.destinationSymmetricKey
HTTP.execute(.post, url, parameters: parameters).done2 { rawResponse in
HTTP.execute(.post, url, body: body).done2 { rawResponse in
guard let json = rawResponse as? JSON, let base64EncodedIVAndCiphertext = json["result"] as? String,
let ivAndCiphertext = Data(base64Encoded: base64EncodedIVAndCiphertext), ivAndCiphertext.count >= EncryptionUtilities.ivSize else { return seal.reject(HTTP.Error.invalidJSON) }
do {
@ -330,14 +350,30 @@ public enum OnionRequestAPI {
}
promise.catch2 { error in // Must be invoked on LokiAPI.workQueue
guard case HTTP.Error.httpRequestFailed(let statusCode, let json) = error else { return }
// Marking all the snodes in the path as unreliable here is aggressive, but otherwise users
// can get stuck with a failing path that just refreshes to the same path.
let path = paths.first { $0.contains(guardSnode) }
path?.forEach { snode in
SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode) // Intentionally don't throw
func handleUnspecificError() {
path?.forEach { snode in
SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode) // Intentionally don't throw
}
dropAllPaths()
dropGuardSnode(guardSnode)
}
let prefix = "Next node not found: "
if let message = json?["result"] as? String, message.hasPrefix(prefix) {
let ed25519PublicKey = message.substring(from: prefix.count)
if let path = path, let snode = path.first(where: { $0.publicKeySet?.ed25519Key == ed25519PublicKey }) {
SnodeAPI.handleError(withStatusCode: statusCode, json: json, forSnode: snode) // Intentionally don't throw
do {
try drop(snode)
} catch {
handleUnspecificError()
}
} else {
handleUnspecificError()
}
} else {
handleUnspecificError()
}
dropAllPaths() // A snode in the path is bad; retry with a different path
dropGuardSnode(guardSnode)
}
return promise
}

@ -0,0 +1,42 @@
public extension Storage {
// MARK: Onion Request Paths
internal static let onionRequestPathCollection = "LokiOnionRequestPathCollection"
internal static func setOnionRequestPaths(_ paths: [OnionRequestAPI.Path], using transaction: YapDatabaseReadWriteTransaction) {
// FIXME: This approach assumes 2 paths of length 3 each. We should do better than this.
guard paths.count == 2 else { return }
let path0 = paths[0]
let path1 = paths[1]
guard path0.count == 3, path1.count == 3 else { return }
let collection = onionRequestPathCollection
transaction.setObject(path0[0], forKey: "0-0", inCollection: collection)
transaction.setObject(path0[1], forKey: "0-1", inCollection: collection)
transaction.setObject(path0[2], forKey: "0-2", inCollection: collection)
transaction.setObject(path1[0], forKey: "1-0", inCollection: collection)
transaction.setObject(path1[1], forKey: "1-1", inCollection: collection)
transaction.setObject(path1[2], forKey: "1-2", inCollection: collection)
}
public static func getOnionRequestPaths() -> [OnionRequestAPI.Path] {
var result: [OnionRequestAPI.Path] = []
read { transaction in
let collection = onionRequestPathCollection
if
let path0Snode0 = transaction.object(forKey: "0-0", inCollection: collection) as? Snode,
let path0Snode1 = transaction.object(forKey: "0-1", inCollection: collection) as? Snode,
let path0Snode2 = transaction.object(forKey: "0-2", inCollection: collection) as? Snode,
let path1Snode0 = transaction.object(forKey: "1-0", inCollection: collection) as? Snode,
let path1Snode1 = transaction.object(forKey: "1-1", inCollection: collection) as? Snode,
let path1Snode2 = transaction.object(forKey: "1-2", inCollection: collection) as? Snode {
result = [ [ path0Snode0, path0Snode1, path0Snode2 ], [ path1Snode0, path1Snode1, path1Snode2 ] ]
}
}
return result
}
internal static func clearOnionRequestPaths(using transaction: YapDatabaseReadWriteTransaction) {
transaction.removeAllObjects(inCollection: onionRequestPathCollection)
}
}

@ -89,7 +89,11 @@ public final class SnodeAPI : NSObject {
return Snode(address: "https://\(address)", port: UInt16(port), publicKeySet: Snode.KeySet(ed25519Key: ed25519PublicKey, x25519Key: x25519PublicKey))
})
// randomElement() uses the system's default random generator, which is cryptographically secure
return snodePool.randomElement()!
if !snodePool.isEmpty {
return snodePool.randomElement()!
} else {
throw SnodeAPIError.randomSnodePoolUpdatingFailed
}
}
}.done2 { snode in
seal.fulfill(snode)

@ -41,17 +41,28 @@ public enum HTTP {
}
// MARK: Main
public static func execute(_ verb: Verb, _ url: String, parameters: JSON? = nil, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
var request = URLRequest(url: URL(string: url)!)
request.httpMethod = verb.rawValue
public static func execute(_ verb: Verb, _ url: String, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
return execute(verb, url, body: nil, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
}
public static func execute(_ verb: Verb, _ url: String, parameters: JSON?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
if let parameters = parameters {
do {
guard JSONSerialization.isValidJSONObject(parameters) else { return Promise(error: Error.invalidJSON) }
request.httpBody = try JSONSerialization.data(withJSONObject: parameters, options: [ .fragmentsAllowed ])
let body = try JSONSerialization.data(withJSONObject: parameters, options: [ .fragmentsAllowed ])
return execute(verb, url, body: body, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
} catch (let error) {
return Promise(error: error)
}
} else {
return execute(verb, url, body: nil, timeout: timeout, useSeedNodeURLSession: useSeedNodeURLSession)
}
}
public static func execute(_ verb: Verb, _ url: String, body: Data?, timeout: TimeInterval = HTTP.timeout, useSeedNodeURLSession: Bool = false) -> Promise<JSON> {
var request = URLRequest(url: URL(string: url)!)
request.httpMethod = verb.rawValue
request.httpBody = body
request.timeoutInterval = timeout
let (promise, seal) = Promise<JSON>.pending()
let urlSession = useSeedNodeURLSession ? seedNodeURLSession : defaultURLSession

@ -53,38 +53,6 @@ public extension OWSPrimaryStorage {
return result
}
// MARK: Onion Request Paths
public func setOnionRequestPaths(_ paths: [OnionRequestAPI.Path], in transaction: YapDatabaseReadWriteTransaction) {
// FIXME: This is a bit of a dirty approach that assumes 2 paths of length 3 each. We should do better than this.
guard paths.count == 2 else { return }
let path0 = paths[0]
let path1 = paths[1]
guard path0.count == 3, path1.count == 3 else { return }
let collection = Storage.onionRequestPathCollection
transaction.setObject(path0[0], forKey: "0-0", inCollection: collection)
transaction.setObject(path0[1], forKey: "0-1", inCollection: collection)
transaction.setObject(path0[2], forKey: "0-2", inCollection: collection)
transaction.setObject(path1[0], forKey: "1-0", inCollection: collection)
transaction.setObject(path1[1], forKey: "1-1", inCollection: collection)
transaction.setObject(path1[2], forKey: "1-2", inCollection: collection)
}
public func getOnionRequestPaths(in transaction: YapDatabaseReadTransaction) -> [OnionRequestAPI.Path] {
let collection = Storage.onionRequestPathCollection
guard
let path0Snode0 = transaction.object(forKey: "0-0", inCollection: collection) as? Snode,
let path0Snode1 = transaction.object(forKey: "0-1", inCollection: collection) as? Snode,
let path0Snode2 = transaction.object(forKey: "0-2", inCollection: collection) as? Snode,
let path1Snode0 = transaction.object(forKey: "1-0", inCollection: collection) as? Snode,
let path1Snode1 = transaction.object(forKey: "1-1", inCollection: collection) as? Snode,
let path1Snode2 = transaction.object(forKey: "1-2", inCollection: collection) as? Snode else { return [] }
return [ [ path0Snode0, path0Snode1, path0Snode2 ], [ path1Snode0, path1Snode1, path1Snode2 ] ]
}
public func clearOnionRequestPaths(in transaction: YapDatabaseReadWriteTransaction) {
transaction.removeAllObjects(inCollection: Storage.onionRequestPathCollection)
}
// MARK: Session Requests
public func setSessionRequestTimestamp(for publicKey: String, to timestamp: Date, in transaction: YapDatabaseReadWriteTransaction) {
transaction.setDate(timestamp, forKey: publicKey, inCollection: Storage.sessionRequestTimestampCollection)

@ -13,7 +13,6 @@
return "LokiSwarmCollection-\(publicKey)"
}
@objc public static let onionRequestPathCollection = "LokiOnionRequestPathCollection"
@objc public static let openGroupCollection = "LokiPublicChatCollection"
@objc public static let openGroupProfilePictureURLCollection = "LokiPublicChatAvatarURLCollection"
@objc public static let openGroupUserCountCollection = "LokiPublicChatUserCountCollection"

Loading…
Cancel
Save