Commit 239a28c0 authored by Nawasan Wisitsingkhon's avatar Nawasan Wisitsingkhon

clone ofp10 into ofp13 and setup ofp13

parent 0f8e3af1
pub mod ofp10;
\ No newline at end of file
pub mod ofp10;
pub mod ofp13;
\ No newline at end of file
use crate::openflow::ofp13::{self, ErrorEvent, Msg, PacketInEvent};
use std::{
io::{Read, Write},
net::TcpStream,
};
use super::{
events::{echo_reply::EchoReplyEvent, EchoRequestEvent},
tcp_listener_handler, MessageMarshal, OfpMsgEvent, Openflow13, OpenflowHeader,
};
pub trait ControllerFrame13
where
Self: 'static,
{
fn ofp(&self) -> Openflow13 {
Openflow13::new()
}
fn packet_in_handler(&mut self, xid: u32, packetin: PacketInEvent, stream: &mut TcpStream);
fn new() -> Self;
fn listener(&self, address: &str)
where
Self: Sized,
Self: Send,
Self: Clone,
{
println!("server run at {}", address);
let _ = tcp_listener_handler(address, self.clone());
}
fn handle_header(&mut self, buf: &mut Vec<u8>) -> Option<(u8, usize, u32)> {
let ofp_header = self.ofp().header_parse(&buf);
match ofp_header {
Ok(header) => Some((header.message(), header.pkt_size(), header.xid())),
Err(_) => None,
}
}
fn request_handler(&mut self, buf: &mut Vec<u8>, stream: &mut TcpStream) {
let ofp = self.ofp();
let (message, pkt_size, xid) = match self.handle_header(buf) {
Some(header) => header,
None => return,
};
let mut payload = vec![0u8; pkt_size];
let _ = stream.read(&mut payload);
let message = ofp.msg_parse(message as u8);
match message {
Msg::Hello => self.hello_handler(xid, stream),
Msg::Error => match ErrorEvent::parse(&payload) {
Ok(error) => self.error_handler(error),
Err(_) => (),
},
Msg::EchoRequest => {
self.echo_request_handler(xid, EchoRequestEvent::new(payload), stream)
}
Msg::PacketIn => match PacketInEvent::parse(&payload) {
Ok(pkt_in) => self.packet_in_handler(xid, pkt_in, stream),
Err(_) => (),
},
_ => (),
}
}
fn send_msg<MSM: MessageMarshal>(&self, msg: MSM, xid: u32, stream: &mut TcpStream) {
let ofp = self.ofp();
let mut header_bytes: Vec<u8> = Vec::new();
let mut body_bytes: Vec<u8> = Vec::new();
msg.marshal(&mut body_bytes);
let ofp_header = ofp.header(msg.msg_usize() as u8, body_bytes.len() as u16, xid);
ofp_header.marshal(&mut header_bytes);
header_bytes.append(&mut body_bytes);
let _ = stream.write_all(&header_bytes);
}
/**
* for handle message
*/
fn hello_handler(&self, xid: u32, stream: &mut TcpStream) {
self.send_msg(self.ofp().fetures_req(), xid, stream);
}
fn error_handler(&self, error: ErrorEvent) {
println!("Error {:?}", error.error_type);
}
fn echo_request_handler(&self, xid: u32, echo: EchoRequestEvent, stream: &mut TcpStream) {
self.send_msg(EchoReplyEvent::new(echo.payload), xid, stream);
}
}
use std::{
io::{BufRead, Cursor, Error},
mem::size_of,
};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use crate::{
etherparser::tools::bits::{bytes_to_mac, mac_to_bytes},
openflow::ofp13::PseudoPort,
};
pub enum ActionType {
Output = 0,
SetVlanId = 1,
SetVlanPCP = 2,
StripVlan = 3,
SetSrcMac = 4,
SetDstMac = 5,
SetIPv4Src = 6,
SetIPv4Des = 7,
SetTos = 8,
SetTpSrc = 9,
SetTpDst = 10,
Enqueue = 11,
}
#[derive(Clone)]
pub enum Action {
Oputput(PseudoPort),
SetDlVlan(Option<u16>),
SetDlVlanPcp(u8),
SetDlSrc(u64),
SetDlDest(u64),
SetIpSrc(u32),
SetIpDes(u32),
SetTos(u8),
SetTpSrc(u16),
SetTpDest(u16),
Enqueue(PseudoPort, u32),
Unparsable,
}
impl Action {
pub fn to_action_code(&self) -> ActionType {
match self {
Action::Oputput(_) => ActionType::Output,
Action::SetDlVlan(_) => ActionType::SetVlanId,
Action::SetDlVlanPcp(_) => ActionType::SetVlanPCP,
Action::SetDlSrc(_) => ActionType::SetSrcMac,
Action::SetDlDest(_) => ActionType::SetDstMac,
Action::SetIpSrc(_) => ActionType::SetIPv4Src,
Action::SetIpDes(_) => ActionType::SetIPv4Des,
Action::SetTos(_) => ActionType::SetTos,
Action::SetTpSrc(_) => ActionType::SetTpSrc,
Action::SetTpDest(_) => ActionType::SetTpDst,
Action::Enqueue(_, _) => ActionType::Enqueue,
Action::Unparsable => panic!("Unparse Action to ActionType"),
}
}
pub fn length(&self) -> usize {
let header = size_of::<(u16, u16)>();
let body = match self {
Action::Oputput(_) => size_of::<(u16, u16)>(),
Action::SetDlVlan(_) => size_of::<(u16, u16)>(),
Action::SetDlVlanPcp(_) => size_of::<(u8, [u8; 3])>(),
Action::SetDlSrc(_) => size_of::<([u8; 6], [u8; 6])>(),
Action::SetDlDest(_) => size_of::<([u8; 6], [u8; 6])>(),
Action::SetIpSrc(_) => size_of::<u32>(),
Action::SetIpDes(_) => size_of::<u32>(),
Action::SetTos(_) => size_of::<(u8, [u8; 3])>(),
Action::SetTpSrc(_) => size_of::<(u16, u16)>(),
Action::SetTpDest(_) => size_of::<(u16, u16)>(),
Action::Enqueue(_, _) => size_of::<(u16, [u8; 6], u32)>(),
Action::Unparsable => 0,
};
header + body
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
let _ = bytes.write_u16::<BigEndian>(self.to_action_code() as u16);
let _ = bytes.write_u16::<BigEndian>(self.length() as u16);
match self {
Action::Oputput(pseudo) => {
pseudo.marshal(bytes);
let _ = bytes.write_u16::<BigEndian>(match pseudo {
PseudoPort::Controller(w) => *w as u16,
_ => 0,
});
}
Action::SetDlVlan(None) => {
let _ = bytes.write_u32::<BigEndian>(0xffff);
}
Action::SetDlVlan(Some(vid)) => {
let _ = bytes.write_u16::<BigEndian>(*vid);
let _ = bytes.write_u16::<BigEndian>(0);
}
Action::SetDlVlanPcp(pcp) => {
let _ = bytes.write_u8(*pcp);
for _ in 0..3 {
let _ = bytes.write_u8(0);
}
}
Action::SetDlSrc(mac) | Action::SetDlDest(mac) => {
let mac = bytes_to_mac(*mac);
for m in mac {
let _ = bytes.write_u8(m);
}
for _ in 0..6 {
let _ = bytes.write_u8(0);
}
}
Action::SetIpSrc(address) | Action::SetIpDes(address) => {
let _ = bytes.write_u32::<BigEndian>(*address);
}
Action::SetTos(n) => {
let _ = bytes.write_u8(*n);
}
Action::SetTpSrc(pt) | Action::SetTpDest(pt) => {
let _ = bytes.write_u16::<BigEndian>(*pt);
let _ = bytes.write_u16::<BigEndian>(0);
}
Action::Enqueue(pp, qid) => {
pp.marshal(bytes);
for _ in 0..6 {
let _ = bytes.write_u8(0);
}
let _ = bytes.write_u32::<BigEndian>(*qid);
}
Action::Unparsable => (),
}
}
pub fn parse_sequence(bytes: &mut Cursor<Vec<u8>>) -> Vec<Action> {
if bytes.get_ref().is_empty() {
vec![]
} else {
if let Ok(action) = Action::parse(bytes) {
let mut v = vec![action];
v.append(&mut Action::parse_sequence(bytes));
v
} else {
vec![]
}
}
}
pub fn parse(bytes: &mut Cursor<Vec<u8>>) -> Result<Action, Error> {
let action_code = bytes.read_u16::<BigEndian>()?;
let _ = bytes.read_u16::<BigEndian>()?;
match action_code {
t if t == (ActionType::Output as u16) => {
let port_code = bytes.read_u16::<BigEndian>()?;
let len = bytes.read_u16::<BigEndian>()?;
Ok(Action::Oputput(PseudoPort::new(
port_code,
Some(len as u64),
)))
}
t if t == (ActionType::SetVlanId as u16) => {
let vid = bytes.read_u16::<BigEndian>()?;
bytes.consume(2);
if vid == 0xffff {
Ok(Action::SetDlVlan(None))
} else {
Ok(Action::SetDlVlan(Some(vid)))
}
}
t if t == (ActionType::SetVlanPCP as u16) => {
let pcp = bytes.read_u8()?;
bytes.consume(3);
Ok(Action::SetDlVlanPcp(pcp))
}
t if t == (ActionType::StripVlan as u16) => {
bytes.consume(4);
Ok(Action::SetDlVlan(None))
}
t if t == (ActionType::SetSrcMac as u16) => {
let mut addr = [0u8; 6];
for i in 0..6 {
addr[i] = bytes.read_u8()?;
}
bytes.consume(6);
Ok(Action::SetDlSrc(mac_to_bytes(addr)))
}
t if t == (ActionType::SetDstMac as u16) => {
let mut addr = [0u8; 6];
for i in 0..6 {
addr[i] = bytes.read_u8()?;
}
bytes.consume(6);
Ok(Action::SetDlDest(mac_to_bytes(addr)))
}
t if t == (ActionType::SetIPv4Src as u16) => {
Ok(Action::SetIpSrc(bytes.read_u32::<BigEndian>()?))
}
t if t == (ActionType::SetIPv4Des as u16) => {
Ok(Action::SetIpDes(bytes.read_u32::<BigEndian>()?))
}
t if t == (ActionType::SetTos as u16) => {
let tos = bytes.read_u8()?;
bytes.consume(3);
Ok(Action::SetTos(tos))
}
t if t == (ActionType::SetTpSrc as u16) => {
let pt = bytes.read_u16::<BigEndian>()?;
bytes.consume(2);
Ok(Action::SetTpSrc(pt))
}
t if t == (ActionType::SetTpDst as u16) => {
let pt = bytes.read_u16::<BigEndian>()?;
bytes.consume(2);
Ok(Action::SetTpDest(pt))
}
t if t == (ActionType::Enqueue as u16) => {
let pt = bytes.read_u16::<BigEndian>()?;
bytes.consume(6);
let qid = bytes.read_u32::<BigEndian>()?;
Ok(Action::Enqueue(PseudoPort::new(pt, Some(0)), qid))
}
_ => Ok(Action::Unparsable),
}
}
}
pub trait SizeCheck {
fn size_of_sequence(&self) -> usize;
fn move_controller_last(&self) -> Vec<Action>;
}
impl SizeCheck for Vec<Action> {
fn size_of_sequence(&self) -> usize {
self.iter().fold(0, |acc, x| x.length() + acc)
}
fn move_controller_last(&self) -> Vec<Action> {
let mut not_ctrl: Vec<Action> = Vec::new();
let mut is_ctrl: Vec<Action> = Vec::new();
for act in self {
match act {
Action::Oputput(PseudoPort::Controller(_)) => {
is_ctrl.push(act.clone());
}
_ => not_ctrl.push(act.clone()),
}
}
not_ctrl.append(&mut is_ctrl);
not_ctrl
}
}
use std::io::Write;
use crate::openflow::ofp13::{self, MessageMarshal, Msg};
pub struct EchoReplyEvent {
pub payload: Vec<u8>,
}
impl EchoReplyEvent {
pub fn new(payload: Vec<u8>) -> Self {
Self { payload }
}
}
impl MessageMarshal for EchoReplyEvent {
fn marshal(&self, bytes: &mut Vec<u8>) {
let _ = bytes.write_all(&self.payload);
}
fn msg_code(&self) -> ofp13::Msg {
Msg::EchoReply
}
fn msg_usize(&self) -> usize {
Msg::EchoReply as usize
}
fn size_of(&self) -> usize {
self.payload.len()
}
}
use std::io::Write;
use crate::openflow::ofp13::{self, MessageMarshal, Msg};
pub struct EchoRequestEvent {
pub payload: Vec<u8>,
}
impl EchoRequestEvent {
pub fn new(payload: Vec<u8>) -> Self {
Self { payload }
}
}
impl MessageMarshal for EchoRequestEvent {
fn marshal(&self, bytes: &mut Vec<u8>) {
let _ = bytes.write_all(&self.payload);
}
fn msg_code(&self) -> ofp13::Msg {
Msg::EchoRequest
}
fn msg_usize(&self) -> usize {
Msg::EchoRequest as usize
}
fn size_of(&self) -> usize {
self.payload.len()
}
}
use std::{
io::{BufRead, Cursor, Error},
mem::size_of,
};
use super::error_type::ErrorType;
use crate::openflow::ofp10::{MessageMarshal, Msg};
use byteorder::{BigEndian, ReadBytesExt};
pub struct ErrorEvent {
pub error_type: ErrorType,
pub payload: Vec<u8>,
}
impl ErrorEvent {
pub fn new(error_type: ErrorType, payload: Vec<u8>) -> Self {
ErrorEvent {
error_type,
payload,
}
}
pub fn parse(buf: &Vec<u8>) -> Result<ErrorEvent, Error> {
let mut bytes = Cursor::new(buf);
let error_type = bytes.read_u16::<BigEndian>()?;
let error_code = bytes.read_u16::<BigEndian>()?;
let code = ErrorType::new(error_type, error_code);
let payload = bytes.fill_buf()?.to_vec();
Ok(ErrorEvent::new(code, payload))
}
}
impl MessageMarshal for ErrorEvent {
fn marshal(&self, _: &mut Vec<u8>) {}
fn msg_code(&self) -> crate::openflow::ofp10::Msg {
Msg::Error
}
fn msg_usize(&self) -> usize {
Msg::Error as usize
}
fn size_of(&self) -> usize {
size_of::<(u16, u16)>() + self.payload.len()
}
}
#[derive(Debug)]
pub enum ErrorType {
HelloFailed(HelloFailed),
BadRequest(BadRequest),
BadAction(BadAction),
FlowModFailed(FlowModFailed),
PortModFailed(PortModFailed),
QueueOpFailed(QueueOpFailed),
}
impl ErrorType {
pub fn new(error_type: u16, error_code: u16) -> ErrorType {
match error_type {
0 => ErrorType::HelloFailed(HelloFailed::new(error_code)),
1 => ErrorType::BadRequest(BadRequest::new(error_code)),
2 => ErrorType::BadAction(BadAction::new(error_code)),
3 => ErrorType::FlowModFailed(FlowModFailed::new(error_code)),
4 => ErrorType::PortModFailed(PortModFailed::new(error_code)),
5 => ErrorType::QueueOpFailed(QueueOpFailed::new(error_code)),
_ => panic!("bad error_type in error {}", error_type),
}
}
}
#[derive(Debug)]
pub enum HelloFailed {
Incompatible,
EPerm,
}
impl HelloFailed {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::Incompatible,
_ => Self::EPerm,
}
}
}
#[derive(Debug)]
pub enum BadRequest {
BadVersion,
BadType,
BadStat,
BadVendor,
BadSubType,
EPerm,
BadLen,
BufferEmpty,
BufferUnkown,
}
impl BadRequest {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::BadVersion,
1 => Self::BadType,
2 => Self::BadStat,
3 => Self::BadVendor,
4 => Self::BadSubType,
5 => Self::EPerm,
6 => Self::BadLen,
7 => Self::BufferEmpty,
8 => Self::BufferUnkown,
_ => Self::BadVersion,
}
}
}
#[derive(Debug)]
pub enum BadAction {
BadType,
BadLen,
BadVendor,
BadVendotType,
BadOutPort,
BadArgument,
EPerm,
TooMany,
BadQueue,
}
impl BadAction {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::BadType,
1 => Self::BadLen,
2 => Self::BadVendor,
3 => Self::BadVendotType,
4 => Self::BadOutPort,
5 => Self::BadArgument,
6 => Self::EPerm,
7 => Self::TooMany,
_ => Self::BadQueue,
}
}
}
#[derive(Debug)]
pub enum FlowModFailed {
AllTablesFull,
Overlap,
EPerm,
BadEmergTimeout,
BadCommand,
Unsupported,
}
impl FlowModFailed {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::AllTablesFull,
1 => Self::Overlap,
2 => Self::EPerm,
3 => Self::BadEmergTimeout,
4 => Self::BadCommand,
_ => Self::Unsupported,
}
}
}
#[derive(Debug)]
pub enum PortModFailed {
BadPort,
BadHwAddr,
}
impl PortModFailed {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::BadPort,
_ => Self::BadHwAddr,
}
}
}
#[derive(Debug)]
pub enum QueueOpFailed {
BadPort,
BadQueue,
EPerm,
}
impl QueueOpFailed {
pub fn new(error_code: u16) -> Self {
match error_code {
0 => Self::BadPort,
1 => Self::BadQueue,
_ => Self::EPerm,
}
}
}
pub mod error_handler;
pub use error_handler::ErrorEvent;
pub mod error_type;
\ No newline at end of file
use crate::openflow::ofp13::{MessageMarshal, Msg};
pub struct FeaturesReqEvent {}
impl FeaturesReqEvent {
pub fn new() -> Self {
FeaturesReqEvent {}
}
}
impl MessageMarshal for FeaturesReqEvent {
fn marshal(&self, _: &mut Vec<u8>) {}
fn msg_code(&self) -> Msg {
Msg::FeaturesRequest
}
fn size_of(&self) -> usize {
0
}
fn msg_usize(&self) -> usize {
Msg::FeaturesRequest as usize
}
}
pub enum FlowModCommand {
Add = 0,
Modify = 1,
ModifyStrict = 2,
Delete = 3,
DeleteStrict = 4,
Unparsable = -1,
}
impl FlowModCommand {
pub fn to_number(&self) -> usize {
match self {
FlowModCommand::Add => Self::Add as usize,
FlowModCommand::Modify => Self::Modify as usize,
FlowModCommand::ModifyStrict => Self::ModifyStrict as usize,
FlowModCommand::Delete => Self::Delete as usize,
FlowModCommand::DeleteStrict => Self::DeleteStrict as usize,
FlowModCommand::Unparsable => Self::Unparsable as usize,
}
}
pub fn parse(byte: u16) -> Self {
match byte {
0 => Self::Add,
1 => Self::Modify,
2 => Self::ModifyStrict,
3 => Self::Delete,
4 => Self::DeleteStrict,
_ => Self::Unparsable,
}
}
}
use byteorder::{BigEndian, WriteBytesExt};
pub struct FlowModFlags {
pub send_flow_rem: bool,
pub check_overlap: bool,
pub emerg: bool,
}
impl FlowModFlags {
pub fn new(send_flow_rem: bool, check_overlap: bool, emerg: bool) -> Self {
Self {
send_flow_rem,
check_overlap,
emerg,
}
}
pub fn all_false() -> Self {
Self {
check_overlap: false,
emerg: false,
send_flow_rem: false,
}
}
pub fn parse(byte: u16) -> Self {
let send_flow_rem = byte >> 0 & 1 != 0;
let check_overlap = byte >> 1 & 1 != 0;
let emerg = byte >> 2 & 1 != 0;
Self {
send_flow_rem,
check_overlap,
emerg,
}
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
let mut value = 0u16;
if self.send_flow_rem {
value |= 1 << 0;
}
if self.check_overlap {
value |= 1 << 1;
}
if self.emerg {
value |= 1 << 2;
}
let _ = bytes.write_u16::<BigEndian>(value);
}
}
use std::io::{Cursor, Error};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use crate::openflow::ofp13::{
events::{actions::SizeCheck, Action},
ofp_port::OfpPort,
MessageMarshal, Msg, PseudoPort,
};
use super::{FlowModCommand, FlowModFlags, MatchFields};
pub enum Timeout {
Permanent,
ExpireAfter(u16),
}
impl Timeout {
pub fn parse(tm: u16) -> Self {
match tm {
0 => Self::Permanent,
d => Timeout::ExpireAfter(d),
}
}
pub fn to_int(&self) -> u16 {
match self {
Timeout::Permanent => 0,
Timeout::ExpireAfter(d) => *d,
}
}
}
pub struct FlowModEvent {
command: FlowModCommand,
match_fields: MatchFields,
priority: u16,
actions: Vec<Action>,
cookie: u64,
idle_timeout: Timeout,
hard_timeout: Timeout,
flags: FlowModFlags,
buffer_id: Option<u32>,
out_port: Option<PseudoPort>,
}
impl FlowModEvent {
pub fn add_flow(
priority: u16,
match_fileds: MatchFields,
actions: Vec<Action>,
buffer_id: Option<u32>,
) -> Self {
Self {
command: FlowModCommand::Add,
match_fields: match_fileds,
priority,
actions,
cookie: 0,
idle_timeout: Timeout::Permanent,
hard_timeout: Timeout::Permanent,
flags: FlowModFlags::all_false(),
buffer_id,
out_port: None,
}
}
pub fn parse(buf: &[u8]) -> Result<FlowModEvent, Error> {
let mut bytes = Cursor::new(buf.to_vec());
let match_fields = MatchFields::parse(&mut bytes)?;
let cookie = bytes.read_u64::<BigEndian>()?;
let command = FlowModCommand::parse(bytes.read_u16::<BigEndian>()?);
let idle_timeout = Timeout::parse(bytes.read_u16::<BigEndian>()?);
let hard_timeout = Timeout::parse(bytes.read_u16::<BigEndian>()?);
let priority = bytes.read_u16::<BigEndian>()?;
let buffer_id = bytes.read_i32::<BigEndian>()?;
let out_port = PseudoPort::parse(bytes.read_u16::<BigEndian>()?);
let flags = bytes.read_u16::<BigEndian>()?;
let actions = Action::parse_sequence(&mut bytes);
Ok(FlowModEvent {
command,
match_fields,
cookie,
actions,
priority,
idle_timeout,
hard_timeout,
flags: FlowModFlags::parse(flags),
buffer_id: {
match buffer_id {
-1 => None,
n => Some(n as u32),
}
},
out_port,
})
}
}
impl MessageMarshal for FlowModEvent {
fn msg_usize(&self) -> usize {
Msg::FlowMod as usize
}
fn size_of(&self) -> usize {
24
}
fn msg_code(&self) -> Msg {
Msg::FlowMod
}
fn marshal(&self, bytes: &mut Vec<u8>) {
self.match_fields.marshal(bytes);
let _ = bytes.write_u64::<BigEndian>(self.cookie);
let _ = bytes.write_u16::<BigEndian>(self.command.to_number() as u16);
let _ = bytes.write_u16::<BigEndian>(self.idle_timeout.to_int());
let _ = bytes.write_u16::<BigEndian>(self.hard_timeout.to_int());
let _ = bytes.write_u16::<BigEndian>(self.priority);
let _ = bytes.write_i32::<BigEndian>(match self.buffer_id {
None => -1,
Some(buf_id) => buf_id as i32,
});
match self.out_port.as_ref() {
Some(p) => p.marshal(bytes),
None => {
let _ = bytes.write_u16::<BigEndian>(OfpPort::None as u16);
}
}
self.flags.marshal(bytes);
for act in self.actions.move_controller_last() {
match act {
Action::Oputput(PseudoPort::Table) => {
panic!("Openflow table not allowed")
}
_ => (),
}
act.marshal(bytes);
}
}
}
use std::io::{BufRead, Cursor, Error};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use crate::etherparser::tools::bits::{bit_bool, bytes_to_mac, mac_to_bytes, set_bit};
pub struct Mask<T> {
pub ip: T,
pub mask: Option<T>,
}
impl Mask<u32> {
pub fn to_int(&self) -> u32 {
match self.mask {
Some(v) => v,
None => 0,
}
}
}
struct Wildcards {
pub in_port: bool,
pub mac_dest: bool,
pub mac_src: bool,
pub ethernet_type: bool,
pub vlan_vid: bool,
pub vlan_pcp: bool,
pub ip_src: u32,
pub ip_dest: u32,
pub protocol: bool,
pub tos: bool,
pub transport_src: bool,
pub transport_dest: bool,
}
impl Wildcards {
pub fn from_match_fields(match_fields: &MatchFields) -> Wildcards {
Wildcards {
in_port: match_fields.in_port.is_none(),
vlan_vid: match_fields.vlan_vid.is_none(),
mac_src: match_fields.mac_src.is_none(),
mac_dest: match_fields.mac_dest.is_none(),
ethernet_type: match_fields.ethernet_type.is_none(),
protocol: match_fields.protocol.is_none(),
transport_src: match_fields.transport_src.is_none(),
transport_dest: match_fields.transport_dest.is_none(),
ip_src: Wildcards::mask_bits(&match_fields.ip_src),
ip_dest: Wildcards::mask_bits(&match_fields.ip_dest),
vlan_pcp: match_fields.vlan_pcp.is_none(),
tos: match_fields.tos.is_none(),
}
}
pub fn parse(byte: u32) -> Wildcards {
Wildcards {
in_port: bit_bool(0, byte),
vlan_vid: bit_bool(1, byte),
mac_src: bit_bool(2, byte),
mac_dest: bit_bool(3, byte),
ethernet_type: bit_bool(4, byte),
protocol: bit_bool(5, byte),
transport_src: bit_bool(6, byte),
transport_dest: bit_bool(7, byte),
ip_src: Wildcards::get_nw_mask(byte, 8),
ip_dest: Wildcards::get_nw_mask(byte, 14),
vlan_pcp: bit_bool(20, byte),
tos: bit_bool(21, byte),
}
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
let mut match_field = 0u32;
match_field = set_bit(match_field, 0, self.in_port);
match_field = set_bit(match_field, 1, self.vlan_vid);
match_field = set_bit(match_field, 2, self.mac_src);
match_field = set_bit(match_field, 3, self.mac_dest);
match_field = set_bit(match_field, 4, self.ethernet_type);
match_field = set_bit(match_field, 5, self.protocol);
match_field = set_bit(match_field, 6, self.transport_src);
match_field = set_bit(match_field, 7, self.transport_dest);
match_field = Wildcards::set_nw_mask(match_field, 8, self.ip_src);
match_field = Wildcards::set_nw_mask(match_field, 14, self.ip_dest);
match_field = set_bit(match_field, 20, self.vlan_pcp);
match_field = set_bit(match_field, 21, self.tos);
let _ = bytes.write_u32::<BigEndian>(match_field);
}
pub fn get_nw_mask(f: u32, offset: usize) -> u32 {
(f >> offset) & 0x3f
}
pub fn set_nw_mask(byte: u32, offset: usize, set: u32) -> u32 {
let value = (0x3f & set) << offset;
byte | value
}
pub fn mask_bits(mask: &Option<Mask<u32>>) -> u32 {
match mask {
None => 32,
Some(m) => match m.mask {
None => 0,
Some(m) => m,
},
}
}
}
pub struct MatchFields {
pub in_port: Option<u16>,
pub mac_dest: Option<u64>,
pub mac_src: Option<u64>,
pub ethernet_type: Option<u16>,
pub vlan_vid: Option<u16>, // vlan type
pub vlan_pcp: Option<u8>,
pub ip_src: Option<Mask<u32>>,
pub ip_dest: Option<Mask<u32>>,
pub protocol: Option<u8>,
pub tos: Option<u8>,
pub transport_src: Option<u16>,
pub transport_dest: Option<u16>,
}
impl MatchFields {
pub fn match_all() -> Self {
Self {
ethernet_type: None,
in_port: None,
ip_dest: None,
ip_src: None,
mac_dest: None,
mac_src: None,
protocol: None,
tos: None,
transport_dest: None,
transport_src: None,
vlan_pcp: None,
vlan_vid: None,
}
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
let wildcard = Wildcards::from_match_fields(self);
wildcard.marshal(bytes);
let _ = bytes.write_u16::<BigEndian>(match self.in_port {
Some(p) => p,
None => 0,
});
let mac_src = match self.mac_src {
Some(mac) => bytes_to_mac(mac),
None => bytes_to_mac(0),
};
for m in mac_src {
let _ = bytes.write_u8(m);
}
let mac_dest = match self.mac_dest {
Some(mac) => bytes_to_mac(mac),
None => bytes_to_mac(0),
};
for m in mac_dest {
let _ = bytes.write_u8(m);
}
let vlan = match self.vlan_vid {
Some(v) => v,
None => 0xffff,
};
let _ = bytes.write_u16::<BigEndian>(vlan);
let _ = bytes.write_u8(match self.vlan_pcp {
Some(v) => v,
None => 0,
});
let _ = bytes.write_u8(0);
let _ = bytes.write_u16::<BigEndian>(match self.ethernet_type {
Some(v) => v,
None => 0,
});
let _ = bytes.write_u8(match self.tos {
Some(v) => v,
None => 0,
});
let _ = bytes.write_u8(match self.protocol {
Some(v) => v,
None => 0,
});
let _ = bytes.write_u16::<BigEndian>(0);
let _ = bytes.write_u32::<BigEndian>(match &self.ip_src {
Some(ip) => ip.ip,
None => 0,
});
let _ = bytes.write_u32::<BigEndian>(match &self.ip_dest {
Some(ip) => ip.ip,
None => 0,
});
let _ = bytes.write_u16::<BigEndian>(match self.transport_src {
Some(v) => v,
None => 0,
});
let _ = bytes.write_u16::<BigEndian>(match self.transport_dest {
Some(v) => v,
None => 0,
});
}
pub fn parse(bytes: &mut Cursor<Vec<u8>>) -> Result<MatchFields, Error> {
let wildcards = Wildcards::parse(bytes.read_u32::<BigEndian>()?);
let in_port = if wildcards.in_port {
None
} else {
Some(bytes.read_u16::<BigEndian>()?)
};
let mac_src = if wildcards.mac_src {
None
} else {
let mut arr: [u8; 6] = [0; 6];
for i in 0..6 {
arr[i] = bytes.read_u8()?;
}
Some(mac_to_bytes(arr))
};
let mac_dest = if wildcards.mac_dest {
None
} else {
let mut arr: [u8; 6] = [0; 6];
for i in 0..6 {
arr[i] = bytes.read_u8()?;
}
Some(mac_to_bytes(arr))
};
let vlan_vid = if wildcards.vlan_vid {
None
} else {
let vid = bytes.read_u16::<BigEndian>()?;
if vid == 0xfff {
None
} else {
Some(bytes.read_u16::<BigEndian>()?)
}
};
let vlan_pcp = if wildcards.vlan_pcp {
None
} else {
Some(bytes.read_u8()?)
};
bytes.consume(1);
let ethernet_type = if wildcards.ethernet_type {
None
} else {
Some(bytes.read_u16::<BigEndian>()?)
};
let tos = if wildcards.tos {
None
} else {
Some(bytes.read_u8()?)
};
let protocol = if wildcards.protocol {
None
} else {
Some(bytes.read_u8()?)
};
bytes.consume(2);
let ip_src = if wildcards.ip_src >= 32 {
None
} else if wildcards.ip_src == 0 {
Some(Mask {
ip: bytes.read_u32::<BigEndian>()?,
mask: None,
})
} else {
Some(Mask {
ip: bytes.read_u32::<BigEndian>()?,
mask: Some(wildcards.ip_src),
})
};
let ip_dest = if wildcards.ip_dest >= 32 {
None
} else if wildcards.ip_dest == 0 {
Some(Mask {
ip: bytes.read_u32::<BigEndian>()?,
mask: None,
})
} else {
Some(Mask {
ip: bytes.read_u32::<BigEndian>()?,
mask: Some(wildcards.ip_dest),
})
};
let transport_src = if wildcards.transport_src {
None
} else {
Some(bytes.read_u16::<BigEndian>()?)
};
let transport_dest = if wildcards.transport_dest {
None
} else {
Some(bytes.read_u16::<BigEndian>()?)
};
Ok(MatchFields {
in_port,
mac_src,
mac_dest,
ethernet_type,
vlan_vid,
vlan_pcp,
ip_src,
ip_dest,
protocol,
tos,
transport_src,
transport_dest,
})
}
}
pub mod flow_mod_handler;
pub use flow_mod_handler::FlowModEvent;
pub mod command;
pub use command::FlowModCommand;
pub mod match_fields;
pub use match_fields::{Mask, MatchFields};
pub mod flow_mod_flags;
pub use flow_mod_flags::FlowModFlags;
\ No newline at end of file
use crate::openflow::ofp13::{MessageMarshal, Msg};
pub struct HelloEvent {}
impl HelloEvent {
pub fn new() -> Self {
HelloEvent {}
}
}
impl MessageMarshal for HelloEvent {
fn marshal(&self, _: &mut Vec<u8>) {}
fn msg_code(&self) -> Msg {
Msg::Hello
}
fn size_of(&self) -> usize {
0
}
fn msg_usize(&self) -> usize {
Msg::Hello as usize
}
}
pub mod error;
pub use error::ErrorEvent;
pub mod packet_in;
pub use packet_in::{PacketInEvent, PacketInReason};
pub mod packet_out;
pub use packet_out::PacketOutEvent;
pub mod flow_mod;
pub use flow_mod::{FlowModCommand, FlowModEvent, FlowModFlags, MatchFields};
pub mod actions;
pub use actions::Action;
pub mod hello;
pub use hello::HelloEvent;
pub mod features_req;
pub use features_req::FeaturesReqEvent;
pub mod payload;
pub use payload::Payload;
pub mod echo_request;
pub use echo_request::EchoRequestEvent;
pub mod echo_reply;
pub use echo_reply::EchoReplyEvent;
use crate::etherparser::ethernet::EthernetFrame;
use super::Payload;
use byteorder::{BigEndian, ReadBytesExt};
use std::io::{BufRead, Cursor, Error};
#[derive(Debug)]
pub enum PacketInReason {
NoMatch,
Action,
InvalidTTL,
Unknown(u8),
}
impl PacketInReason {
fn new(code: u8) -> Self {
match code {
0 => PacketInReason::NoMatch,
1 => PacketInReason::Action,
2 => PacketInReason::InvalidTTL,
t => PacketInReason::Unknown(t),
}
}
}
pub struct PacketInEvent {
pub buf_id: Option<u32>,
pub total_len: u16,
pub in_port: u16,
pub reason: PacketInReason,
pub table_id: u8,
pub payload: Payload,
}
impl PacketInEvent {
pub fn ether_parse(&self) -> Result<EthernetFrame, Error> {
match &self.payload {
Payload::Buffered(_, p) | Payload::NoBuffered(p) => EthernetFrame::parse(&p),
}
}
pub fn parse(payload: &Vec<u8>) -> Result<PacketInEvent, Error> {
let mut bytes = Cursor::new(payload.to_vec());
let buf_id = match bytes.read_i32::<BigEndian>()? {
-1 => None,
n => Some(n as u32),
};
let total_len = bytes.read_u16::<BigEndian>()?;
let in_port = bytes.read_u16::<BigEndian>()?;
let reason = PacketInReason::new(bytes.read_u8()?);
let table_id = bytes.read_u8()?;
let packet = bytes.fill_buf()?.to_vec();
let payload = match buf_id {
Some(n) => Payload::Buffered(n as u32, packet),
None => Payload::NoBuffered(packet),
};
Ok(PacketInEvent {
buf_id,
total_len,
in_port,
reason,
table_id,
payload,
})
}
}
use std::{
io::{BufRead, Cursor, Error, Read},
mem::size_of,
};
use crate::openflow::ofp13::PseudoPort;
use crate::openflow::ofp13::{ofp_port::OfpPort, MessageMarshal, Msg};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use super::{actions::SizeCheck, Action, Payload};
pub struct PacketOutEvent {
pub payload: Payload,
pub in_port: Option<u16>,
pub actions: Vec<Action>,
}
impl MessageMarshal for PacketOutEvent {
fn marshal(&self, bytes: &mut Vec<u8>) {
let _ = bytes.write_i32::<BigEndian>(match self.payload {
Payload::Buffered(n, _) => n as i32,
Payload::NoBuffered(_) => -1,
});
match self.in_port {
Some(id) => {
PseudoPort::PhysicalPort(id).marshal(bytes);
}
None => {
let _ = bytes.write_u16::<BigEndian>(OfpPort::None as u16);
}
}
let _ = bytes.write_u16::<BigEndian>(self.actions.size_of_sequence() as u16);
for act in self.actions.move_controller_last() {
act.marshal(bytes);
}
self.payload.marshal(bytes);
}
fn msg_code(&self) -> Msg {
Msg::PacketOut
}
fn msg_usize(&self) -> usize {
Msg::PacketOut as usize
}
fn size_of(&self) -> usize {
size_of::<(u32, u16, u16)>() + self.actions.size_of_sequence() + self.payload.length()
}
}
impl PacketOutEvent {
pub fn new(in_port: Option<u16>, payload: Payload, actions: Vec<Action>) -> Self {
Self {
in_port,
payload,
actions,
}
}
pub fn parse(buf: &Vec<u8>) -> Result<Self, Error> {
let mut bytes = Cursor::new(buf);
let buf_id = match bytes
.read_i32::<BigEndian>()
.expect("cannot parse buf id in packetout")
{
-1 => None,
n => Some(n),
};
let in_port = bytes.read_u16::<BigEndian>()?;
let action_len = bytes.read_u16::<BigEndian>()?;
let mut actions_buf = vec![0; action_len as usize];
let _ = bytes.read_exact(&mut actions_buf);
let mut action_bytes = Cursor::new(actions_buf);
let actions = Action::parse_sequence(&mut action_bytes);
Ok(Self {
payload: match buf_id {
None => Payload::NoBuffered(bytes.fill_buf()?.to_vec()),
Some(n) => Payload::Buffered(n as u32, bytes.fill_buf()?.to_ascii_lowercase()),
},
in_port: {
if in_port == OfpPort::None as u16 {
None
} else {
Some(in_port)
}
},
actions,
})
}
}
use std::io::Write;
pub enum Payload {
Buffered(u32, Vec<u8>),
NoBuffered(Vec<u8>),
}
impl Payload {
pub fn length(&self) -> usize {
match self {
Payload::Buffered(_, p) | Payload::NoBuffered(p) => p.len(),
}
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
match self {
Payload::Buffered(_, buf) | Payload::NoBuffered(buf) => {
let _ = bytes.write_all(&buf);
}
}
}
}
use std::mem::transmute;
#[derive(Clone)]
pub enum Msg {
Hello = 0,
Error = 1,
EchoRequest = 2,
EchoReply = 3,
Vendor = 4,
FeaturesRequest = 5,
FeaturesReply = 6,
ConfigRequest = 7,
ConfigReply = 8,
SetConfig = 9,
PacketIn = 10,
FlowRemove = 11,
PortStatus = 12,
PacketOut = 13,
FlowMod = 14,
PortMod = 15,
StatsRequest = 16,
StateReply = 17,
BarrierRequest = 18,
BarrierReply = 19,
QueueGetConfigRequest = 20,
QueueGetConfigReply = 21,
NotFound = 0xff,
}
impl Msg {
pub fn to_int(&self) -> u8 {
self.clone() as u8
}
pub fn from(msg_code: u8) -> Self {
if msg_code > 21 {
return Self::NotFound;
}
unsafe { transmute::<u8, Msg>(msg_code) }
}
}
pub mod message;
pub use message::Msg;
pub mod ofp_port;
pub use ofp_port::PseudoPort;
pub mod events;
pub use events::{
Action, EchoReplyEvent, EchoRequestEvent, ErrorEvent, FlowModEvent, HelloEvent, MatchFields,
PacketInEvent, PacketOutEvent,
};
pub mod ofp_header;
pub use ofp_header::OfpHeader;
pub mod ofp_manager;
pub use ofp_manager::Openflow13;
pub mod controller_frame;
pub use controller_frame::ControllerFrame13;
pub mod tcp_listener;
pub use tcp_listener::tcp_listener_handler;
pub mod traiter;
pub use traiter::{MessageMarshal, OfpMsgEvent, OpenflowHeader};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use std::{
io::{Cursor, Error},
mem::size_of,
};
use crate::openflow::ofp13::OpenflowHeader;
pub struct OfpHeader {
pub version: u8,
pub message: u8,
pub length: u16,
pub xid: u32,
}
impl OpenflowHeader for OfpHeader {
fn new(message: u8, length: usize, xid: usize) -> Self {
Self {
version: 1,
message,
length: (size_of::<OfpHeader>() + length) as u16,
xid: xid as u32,
}
}
fn version(&self) -> usize {
1
}
fn message(&self) -> u8 {
self.message
}
fn length(&self) -> usize {
self.length as usize
}
fn xid(&self) -> u32 {
self.xid
}
fn header_size(&self) -> usize {
size_of::<Self>()
}
fn pkt_size(&self) -> usize {
return self.length as usize - size_of::<Self>();
}
fn parse(buf: &Vec<u8>) -> Result<Self, Error> {
let mut buf_cursor = Cursor::new(buf);
let version = buf_cursor.read_u8()?;
let message = buf_cursor.read_u8()?;
let length = buf_cursor.read_u16::<BigEndian>()?;
let xid = buf_cursor.read_u32::<BigEndian>()?;
Ok(Self {
version,
message,
length,
xid,
})
}
fn marshal(&self, bytes: &mut Vec<u8>) {
let _ = bytes.write_u8(self.version);
let _ = bytes.write_u8(self.message);
let _ = bytes.write_u16::<BigEndian>(self.length);
let _ = bytes.write_u32::<BigEndian>(self.xid);
}
}
use super::{
events::{Action, FeaturesReqEvent, Payload},
ofp_header::OfpHeader,
HelloEvent, Msg, OfpMsgEvent, OpenflowHeader, PacketOutEvent,
};
pub struct Openflow13 {}
impl Openflow13 {
pub fn new() -> Self {
Openflow13 {}
}
}
impl OfpMsgEvent for Openflow13 {
fn header_parse(&self, bytes: &Vec<u8>) -> Result<OfpHeader, std::io::Error> {
OfpHeader::parse(bytes)
}
fn header_size(&self) -> usize {
8
}
fn hello_event(&self) -> HelloEvent {
HelloEvent::new()
}
fn fetures_req(&self) -> FeaturesReqEvent {
FeaturesReqEvent::new()
}
fn packet_out(
&self,
port_id: Option<u16>,
payload: Payload,
actions: Vec<Action>,
) -> PacketOutEvent {
PacketOutEvent::new(port_id, payload, actions)
}
fn ofp_version() -> usize {
1
}
fn version(&self) -> usize {
1
}
fn header(&self, message: u8, length: u16, xid: u32) -> OfpHeader {
OfpHeader::new(message, length as usize, xid as usize)
}
fn msg_parse(&self, msg: u8) -> Msg {
Msg::from(msg)
}
fn msg_usize(&self, msg: Msg) -> usize {
msg.to_int() as usize
}
}
use byteorder::{BigEndian, WriteBytesExt};
pub enum OfpPort {
Max = 0xff00,
InPort = 0xfff8,
Table = 0xfff9,
Normal = 0xfffa,
Flood = 0xfffb,
All = 0xfffc,
Controller = 0xfffd,
Local = 0xfffe,
None = 0xffff,
}
#[derive(Clone)]
pub enum PseudoPort {
PhysicalPort(u16),
InPort,
Table,
Normal,
Flood,
AllPorts,
Controller(u64),
Local,
Unsupport,
}
impl PseudoPort {
pub fn parse(byte: u16) -> Option<PseudoPort> {
if (OfpPort::None as u16) == byte {
None
} else {
Some(PseudoPort::new(byte, Some(0)))
}
}
pub fn new(port: u16, len: Option<u64>) -> PseudoPort {
match port {
p if p == (OfpPort::InPort as u16) => PseudoPort::InPort,
p if p == (OfpPort::Table as u16) => PseudoPort::Table,
p if p == (OfpPort::Normal as u16) => PseudoPort::Normal,
p if p == (OfpPort::Flood as u16) => PseudoPort::Flood,
p if p == (OfpPort::All as u16) => PseudoPort::AllPorts,
p if p == (OfpPort::Controller as u16) => match len {
Some(len) => PseudoPort::Controller(len),
None => PseudoPort::Unsupport,
},
p if p == (OfpPort::Local as u16) => PseudoPort::InPort,
_ => {
if port <= (OfpPort::Max as u16) {
PseudoPort::PhysicalPort(port)
} else {
PseudoPort::Unsupport
}
}
}
}
pub fn marshal(&self, bytes: &mut Vec<u8>) {
let port = match *self {
PseudoPort::PhysicalPort(p) => p,
PseudoPort::InPort => OfpPort::InPort as u16,
PseudoPort::Table => OfpPort::Table as u16,
PseudoPort::Normal => OfpPort::Normal as u16,
PseudoPort::Flood => OfpPort::Flood as u16,
PseudoPort::AllPorts => OfpPort::All as u16,
PseudoPort::Controller(_) => OfpPort::Controller as u16,
PseudoPort::Local => OfpPort::Local as u16,
// not sure how to handle unsupport
PseudoPort::Unsupport => OfpPort::Flood as u16,
};
let _ = bytes.write_u16::<BigEndian>(port);
}
}
use std::{io::Read, net::TcpListener, thread};
use crate::openflow::ofp13::HelloEvent;
use super::{ControllerFrame13, OfpMsgEvent};
pub fn tcp_listener_handler(
address: &str,
controller: impl ControllerFrame13 + Send + 'static + Clone,
) -> Result<(), std::io::Error> {
let listener = TcpListener::bind(address)?;
// let controller = Arc::new(Mutex::new(controller));
for stream in listener.incoming() {
match stream {
Ok(mut stream) => {
if let Ok(addr) = stream.peer_addr() {
println!("server has connection from {}", addr);
}
let mut ctrl = controller.clone();
thread::spawn(move || {
ctrl.send_msg(HelloEvent::new(), 0, &mut stream);
let ofp_size = ctrl.ofp().header_size();
let mut buffer = vec![0u8; ofp_size];
loop {
match stream.read(&mut buffer) {
Ok(v) if v > 0 => {
ctrl.request_handler(&mut buffer, &mut stream);
}
Ok(_) => {
continue;
}
Err(_) => {
println!("cannot read packet");
break;
}
}
}
});
}
Err(_) => panic!("Connection failed!"),
}
}
Ok(())
}
use std::io::Error;
use crate::openflow::ofp13::{
events::{Action, FeaturesReqEvent, HelloEvent, PacketOutEvent, Payload},
ofp_header::OfpHeader,
Msg,
};
/**
* the trait for parse value to bytes.
* for use with Controller's send_msg.
*/
pub trait MessageMarshal {
fn marshal(&self, bytes: &mut Vec<u8>);
fn msg_code(&self) -> Msg;
fn msg_usize(&self) -> usize;
fn size_of(&self) -> usize;
}
/**
* for works with controller to create OfpMsgEvent
*/
pub trait OfpMsgEvent {
fn header(&self, message: u8, length: u16, xid: u32) -> OfpHeader;
fn header_parse(&self, bytes: &Vec<u8>) -> Result<OfpHeader, Error>;
fn version(&self) -> usize;
fn ofp_version() -> usize;
fn header_size(&self) -> usize;
fn msg_usize(&self, msg: Msg) -> usize;
fn msg_parse(&self, msg: u8) -> Msg;
fn hello_event(&self) -> HelloEvent;
fn fetures_req(&self) -> FeaturesReqEvent;
fn packet_out(
&self,
port_id: Option<u16>,
payload: Payload,
actions: Vec<Action>,
) -> PacketOutEvent;
}
use std::io::Error;
pub trait OpenflowHeader {
fn version(&self) -> usize;
fn message(&self) -> u8;
fn length(&self) -> usize;
fn xid(&self) -> u32;
fn pkt_size(&self) -> usize;
fn new(message: u8, length: usize, xid: usize) -> Self;
fn header_size(&self) -> usize;
fn parse(buf: &Vec<u8>) -> Result<Self, Error>
where
Self: Sized;
fn marshal(&self, bytes: &mut Vec<u8>);
}
pub mod header_trait;
pub use header_trait::OpenflowHeader;
pub mod event_trait;
pub use event_trait::{MessageMarshal, OfpMsgEvent};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment