File Revert-rxe-Remove-rxe_cfg.patch of Package rdma-core.18742
commit fc13d676d07e5047177125cd3131c371eec8677b
Author: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
Date: Tue Mar 9 11:24:04 2021 +0100
Revert "rxe: Remove rxe_cfg"
This reverts commit 0d2ff0e1502ebc63346bc9ffd37deb3c4fd0dbc9.
diff --git Documentation/rxe.md Documentation/rxe.md
index ea11fc4b2a27..8e753decb3cb 100644
--- Documentation/rxe.md
+++ Documentation/rxe.md
@@ -1,14 +1,22 @@
# Configure Soft-RoCE (RXE):
+Load rdma_rxe kernel module using the rxe_cfg script included in the librxe RPM:
+
+ # rxe_cfg start (this might require sudo or root privileges)
+
Create RXE device over network interface (e.g. eth0):
- # rdma link add rxe_eth0 type rxe netdev eth0
+ # rxe_cfg add eth0
Use the status command to display the current configuration:
+rxe_cfg status
+
+If configured successfully, you should see output similar to the following:
- # rdma link
+```
+ Name Link Driver Speed NMTU IPv4_addr RDEV RMTU
+ eth0 yes mlx4_en rxe0 1024 (3)
+```
-If you are using a Mellanox HCA, make sure that the mlx4_ib/mlx5_ib kernel
-module is not loaded (modprobe –rv mlx4_ib) in the soft-RoCE machine. Now you
-have an Infiniband device called “rxe0_eth0” that can be used to run any RoCE
-app.
+If you are using a Mellanox HCA: Need to make sure that the mlx4_ib kernel module is not loaded (modprobe –rv mlx4_ib) in the soft-RoCE machine.
+Now you have an Infiniband device called “rxe0” that can be used to run any RoCE app.
diff --git README.md README.md
index b31d448264e5..01d0f6a1c014 100644
--- README.md
+++ README.md
@@ -116,6 +116,8 @@ command above to work.
You can use either `ibv_devices` or `rdma link` to verify that the device was
successfully added.
+Use of `rxe_cfg` is deprecated, please use the `rdma` command instead.
+
# Reporting bugs
Bugs should be reported to the <linux-rdma@vger.kernel.org> mailing list
diff --git debian/rdma-core.install debian/rdma-core.install
index 564d4a1b6353..c60a76cfe416 100644
--- debian/rdma-core.install
+++ debian/rdma-core.install
@@ -21,6 +21,7 @@ lib/udev/rules.d/90-iwpmd.rules
lib/udev/rules.d/90-rdma-hw-modules.rules
lib/udev/rules.d/90-rdma-ulp-modules.rules
lib/udev/rules.d/90-rdma-umad.rules
+usr/bin/rxe_cfg
usr/lib/truescale-serdes.cmds
usr/sbin/iwpmd
usr/sbin/rdma-ndd
@@ -33,3 +34,4 @@ usr/share/man/man5/iwpmd.conf.5
usr/share/man/man7/rxe.7
usr/share/man/man8/iwpmd.8
usr/share/man/man8/rdma-ndd.8
+usr/share/man/man8/rxe_cfg.8
diff --git providers/rxe/CMakeLists.txt providers/rxe/CMakeLists.txt
index d8f3265176e4..15dc627e4e67 100644
--- providers/rxe/CMakeLists.txt
+++ providers/rxe/CMakeLists.txt
@@ -1,3 +1,8 @@
rdma_provider(rxe
rxe.c
)
+rdma_subst_install(FILES "rxe_cfg.in"
+ RENAME "rxe_cfg"
+ DESTINATION "${CMAKE_INSTALL_BINDIR}"
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE
+ )
diff --git providers/rxe/man/CMakeLists.txt providers/rxe/man/CMakeLists.txt
index 53d78dbe7de1..69e8bd8cf20c 100644
--- providers/rxe/man/CMakeLists.txt
+++ providers/rxe/man/CMakeLists.txt
@@ -1,3 +1,4 @@
rdma_man_pages(
rxe.7
+ rxe_cfg.8
)
diff --git providers/rxe/man/rxe.7 providers/rxe/man/rxe.7
index 474ffff120a7..79349e9d3b50 100644
--- providers/rxe/man/rxe.7
+++ providers/rxe/man/rxe.7
@@ -6,7 +6,7 @@ rxe \- Software RDMA over Ethernet
.SH "SYNOPSIS"
\fBmodprobe rdma_rxe\fR
.br
-This is usually performed by a configuration utility (see \fBrdma link\fR(8).)
+This is usually performed by a configuration utility (see \fBrxe_cfg\fR(8).)
.SH "DESCRIPTION"
The rdma_rxe kernel module provides a software implementation of the RoCEv2
@@ -23,9 +23,17 @@ In particular, while the use of a GRH header is optional within IB subnets, it i
\fB/sys/class/infiniband/rxe[0,1,...]\fR
Directory that holds RDMA device information. The format is the same as other RDMA devices.
+.TP
+\fB/sys/module/rdma_rxe_net/parameters/add\fR
+Write only file used by \fBrxe_cfg(8)\fR to add new RXE devices to existing Ethernet devices.
+
+.TP
+\fB/sys/module/rdma_rxe_net/parameters/remove\fR
+Write only file used by \fBrxe_cfg(8)\fR to remove RXE devices.
+
.TP
\fB/sys/module/rdma_rxe_net/parameters/mtu\fR
-Write only file used to configure RoCE and Ethernet MTU values.
+Write only file used by \fBrxe_cfg(8)\fR to configure RoCE and Ethernet MTU values.
.TP
\fB/sys/module/rdma_rxe/parameters/max_ucontext\fR
@@ -90,7 +98,7 @@ Read/Write file that controls the maximum gap between the PSN of request packets
Read/Write file that controls the default mtu used for UD packets.
.SH "SEE ALSO"
-.BR rdma (8),
+.BR rxe_cfg (8),
.BR verbs (7),
.SH "AUTHORS"
diff --git providers/rxe/man/rxe_cfg.8 providers/rxe/man/rxe_cfg.8
new file mode 100644
index 000000000000..c29c3d009306
--- /dev/null
+++ providers/rxe/man/rxe_cfg.8
@@ -0,0 +1,104 @@
+.\" -*- nroff -*-
+.\"
+.TH RXE_CFG 8 2011-06-29 1.0.0
+.SH "NAME"
+rxe_cfg \- rxe configuration tool for RXE (Soft RoCE)
+.SH "SYNOPSIS"
+\fBrxe_cfg [status]\fR
+.br
+\fBrxe_cfg start\fR [\fB\-p\fR \fIproto\fR]
+.br
+\fBrxe_cfg stop\fR
+.br
+\fBrxe_cfg persistent\fR
+.br
+\fBrxe_cfg add\fR [\fB\-n\fR] \fIethN\fR
+.br
+\fBrxe_cfg remove\fR [\fB\-n\fR] \fIethN\fR|\fIrxeN\fR
+.br
+\fBrxe_cfg crc enable\fR|\fBdisable\fR
+.br
+\fBrxe_cfg\fR \fBmtu\fR [\fB\-f\fR] [\fIrxeN\fR] \fImtu_size\fR
+.br
+.SH "DESCRIPTION"
+
+.I
+Note
+This command is deprecated. Please use
+.B
+rdma-link(8)
+instead.
+
+rxe_cfg is the configuration tool for the RXE software implementation of the RoCE protocol.
+
+The RXE kernel modules are loaded, configured, reconfigured and unloaded via the various rxe_cfg command options, documented below.
+
+.SH "PARAMETERS"
+.TP
+\fIproto\fR
+Ethertype field. Default value is 0x8915. This value must be changed to use RXE on Mellanox ConnectX adapters.
+
+.TP
+\fIethN\fR
+Network device name as listed in /sys/class/net. Only Ethernet devices are supported; ie. eth0 or eth0.1234 for VLANs.
+
+.TP
+\fIrxeN\fR
+RXE device name as listed in /sys/class/infiniband/. Examples are rxe0 or rxe1.
+
+.TP
+\fImtu_size\fR
+RoCE mtu. For RoCE the mtu represents the payload excluding headers and has the possible values: 256, 512, 1024, 2048 and 4096.
+
+.SH "COMMANDS"
+.TP
+[\fBstatus\fR]
+The \fBstatus\fR command prints a table of information on available Ethernet devices and configured RXE instances. The status display is the default if no options are provided.
+
+.TP
+\fBstart\fR [\fB\-p\fR \fIproto\fR]
+The \fBstart\fR command loads the RXE modules and configures any persistent instances. If the \fB-p\fR \fIproto\fR option is included, the RXE modules will be configured to use Ethertype = \fIproto\fR. (This allows testing RXE on devices, like Mellanox ConnectX, that already support the default RoCE Ethertype in hardware.)
+
+.TP
+\fBstop\fR
+The \fBstop\fR command unconfigures all RXE instances and attempts to unload the kernel modules.
+
+.TP
+\fBpersistent\fR
+The \fBpersistent\fR command prints the list of Ethernet devices for which a RXE instance is persistently configured.
+
+.TP
+\fBadd\fR [\fB\-n\fR] \fIethN\fR
+The \fBadd\fR command will configure a RXE instance on Ethernet device \fIethN\fR (e.g. eth0). The RXE modules must have already been loaded via \fBrxe_cfg start\fR.
+
+The default behavior is to add \fIethN\fR to a file of persistent configurations and the same RXE device will be configured the next time that \fBrxe_cfg start\fR is run. If the \fB-n\fR option is included the device is not added to the persistence file.
+
+.TP
+\fBremove\fR [\fB\-n\fR] \fIethN\fR|\fIrxeN\fR
+The \fBremove\fR command will remove the specified RXE instance. The parameter must match a currently active ethN or rxeN name.
+
+If the \fB-n\fR option is included the RXE device will be removed but not removed from the persistent state. So it will be recreated the next time that \fBrxe_cfg start\fR is run.
+
+.TP
+\fBmtu\fR [\fB\-f\fR] [\fIrxeN\fR] \fImtu_size\fR
+The \fBmtu\fR command will set the RoCE MTU of all RXE devices to \fImtu_size\fR, provided that the underlying Ethernet MTU is sufficiently large. If the Ethernet MTU is not large enough, RXE will use the largest MTU that fits; the driver remembers the requested RoCE MTU and will increase the currently active MTU if the Ethernet MTU is later changed up to the requested MTU.
+
+If the \fB-f\fR option is included the underlying Ethernet MTUs will be increased if necessary to the minimum size to accommodate a RoCE MTU of \fIsize\fR.
+
+If a \fIrxeN\fR instance is specified then only that instance will be affected by the command.
+
+.TP
+\fBcrc\fR \fBenable\fR|\fBdisable\fR
+The \fBcrc\fR command will enable or disable RoCE ICRC calculation. Valid ICRCs are be required to communicate hardware RoCE NICs, but when a RXE instance is communicating with another RXE instance the performance will be better with the CRC disabled.
+
+This option is global to all RXE instances.
+
+.SH "FILES"
+.TP
+\fB[PREFIX]/etc/rxe.conf\fR
+RXE configuration file. Contains the list of persistent RXE instances. All persistent RXE instances can be removed by deleting this file (note this will take effect on the next "rxe_cfg start" -- to remove actively configured instances, you must "rxe_cfg stop").
+
+.SH "SEE ALSO"
+.BR rxe (7),
+.SH "AUTHORS"
+Written by John Groves, Frank Zago and Bob Pearson at System Fabric Works.
diff --git providers/rxe/rxe_cfg.in providers/rxe/rxe_cfg.in
new file mode 100755
index 000000000000..96126d3ae653
--- /dev/null
+++ providers/rxe/rxe_cfg.in
@@ -0,0 +1,677 @@
+#!/usr/bin/perl
+
+# * Copyright (c) 2009-2011 Mellanox Technologies Ltd. All rights reserved.
+# * Copyright (c) 2009-2011 System Fabric Works, Inc. All rights reserved.
+# *
+# * This software is available to you under a choice of one of two
+# * licenses. You may choose to be licensed under the terms of the GNU
+# * General Public License (GPL) Version 2, available from the file
+# * COPYING in the main directory of this source tree, or the
+# * OpenIB.org BSD license below:
+# *
+# * Redistribution and use in source and binary forms, with or
+# * without modification, are permitted provided that the following
+# * conditions are met:
+# *
+# * - Redistributions of source code must retain the above
+# * copyright notice, this list of conditions and the following
+# * disclaimer.
+# *
+# * - Redistributions in binary form must reproduce the above
+# * copyright notice, this list of conditions and the following
+# * disclaimer in the documentation and/or other materials
+# * provided with the distribution.
+# *
+# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# * SOFTWARE.
+#
+
+use warnings;
+use strict;
+
+use File::Basename;
+use File::Path qw(make_path);
+use Getopt::Long;
+
+my $help = 0;
+my $no_persist = 0;
+my $debug = 0;
+my $force = 0;
+my $linkonly = 0;
+my $parms = "/sys/module/rdma_rxe/parameters";
+my $modprobe_opt = "";
+my $modprobe_checked = "0";
+my $persistence_path = "@CMAKE_INSTALL_FULL_SHAREDSTATEDIR@/rxe";
+my $persistence_file = "${persistence_path}/rxe";
+my $num_persistent = 0;
+my $sys = "/sys/module/rdma_rxe/parameters";
+my %rxe_names;
+my @rxe_array;
+my %eth_names;
+my @eth_list;
+my %eth_driver;
+my %link_state;
+my %link_speed;
+my %eth_mtu;
+my %ipv4_addr;
+my %rxe_mtu;
+my @persistence_array;
+my %persistence_hash;
+my @mlx4_port;
+my @mlx4_ether;
+my @roce_list;
+
+# Read a file and return its contents as a string.
+sub read_file {
+ my $filename = shift;
+ my $result = "";
+
+ if (open(FILE, $filename)) {
+ $result = <FILE>;
+ close FILE;
+ }
+ return $result;
+}
+
+#get mapping between rxe and eth devices
+sub get_names {
+ my $i = 0;
+
+ foreach my $rxe (glob("/sys/class/infiniband/rxe*")) {
+ $rxe = basename($rxe);
+ my $eth = read_file("/sys/class/infiniband/$rxe/parent");
+ chomp($eth);
+
+ if (($eth =~ /[\w]+[\d]/)
+ && ($rxe =~ /rxe[0123456789]/)) {
+
+ # hash ethername to rxename
+ $rxe_names{$eth} = $rxe;
+ $rxe_array[$i++] = $rxe;
+
+ # hash rxename to ethername
+ $eth_names{$rxe} = $eth;
+ }
+ }
+}
+
+# get list of Mellanox RoCE ports
+sub get_mlx4_list {
+ my $i = 0;
+
+ foreach my $mlx4 (glob("/sys/class/infiniband/mlx4_*")) {
+ $mlx4 = basename($mlx4);
+ foreach my $port (glob("/sys/class/infiniband/$mlx4/ports/*")) {
+ $port = basename($port);
+ my $link = read_file("$port/link_layer");
+ chomp($link);
+
+ if ($link =~ "Ethernet") {
+ $roce_list[$i++] = "$mlx4:$port";
+ }
+ }
+ }
+}
+
+#collect per device information
+sub get_dev_info {
+ my @list;
+ my @fields;
+ my @lines;
+ my $line;
+ my $eth;
+ my $drv;
+ my $np;
+ my $i = 0;
+ my $j = 0;
+
+ get_mlx4_list();
+
+ my @my_eth_list = ();
+ foreach my $my_eth_dev (glob("/sys/class/net/*")) {
+ $my_eth_dev = basename($my_eth_dev);
+ if ($my_eth_dev ne "bonding_masters"){
+ my $my_dev_type = read_file("/sys/class/net/${my_eth_dev}/type");
+ chomp($my_dev_type);
+ if ($my_dev_type == "1") {
+ push(@my_eth_list, "$my_eth_dev");
+ }
+ }
+ }
+
+ @list = @my_eth_list;
+ foreach $eth (@list) {
+ chomp($eth);
+
+ $eth_list[$i++] = $eth;
+
+ @lines = `ethtool -i $eth`;
+ foreach $line (@lines) {
+ chomp($line);
+
+ @fields = split(/\s+/, $line);
+ chomp($fields[0]);
+
+ if ($fields[0] =~ /driver:/) {
+ $drv = $fields[1];
+ $eth_driver{$eth} = $drv;
+
+ if ($drv =~ /mlx4_en/ && scalar(@roce_list) > 0 ) {
+ $eth_names{$roce_list[$j++]} = $eth;
+ }
+ }
+ }
+
+ # get link status
+ $link_state{$eth} = "";
+ $link_speed{$eth} = "";
+
+ @lines = `ethtool $eth`;
+ foreach $line (@lines) {
+ chomp($line);
+
+ @fields = split(/:/, $line);
+ if (defined($fields[1])) {
+ $fields[1] =~ s/^\s+//g;
+ if ($fields[0] =~ "Link detected") {
+ $link_state{$eth} = $fields[1];
+ }
+ }
+ elsif ($line =~ "10000baseT") {
+ $link_speed{$eth} = "10GigE";
+ }
+ }
+
+ $ipv4_addr{$eth} = " ";
+ $eth_mtu{$eth} = "";
+
+ @lines = `ip addr show $eth`;
+ foreach $line (@lines) {
+ # get IP address
+ if ($line =~ /inet /) {
+ $line =~ s/^\s+inet ([0-9.]+)\//$1 /g;
+ @fields = split(/\s+/, $line);
+ $ipv4_addr{$eth} = $fields[0];
+ }
+
+ # get ethernet mtu
+ if ($line =~ /mtu /) {
+ $line =~ s/^.*mtu //g;
+ @fields = split(/\s+/, $line);
+ $eth_mtu{$eth} = $fields[0];
+ }
+ }
+ }
+
+ # get rxe mtu
+ foreach my $rxe (@rxe_array) {
+
+ @lines = `ibv_devinfo -d $rxe`;
+ foreach $line (@lines) {
+ if ($line =~ "active_mtu") {
+ $line =~ s/^\s+active_mtu:\s+//g;
+ chomp($line);
+
+ $rxe_mtu{$rxe} = $line;
+ }
+ }
+ $rxe_mtu{$rxe} = "(?)" if (!$rxe_mtu{$rxe});
+ }
+}
+
+# return string or the string "###" if string is all whitespace
+sub set_field {
+ my $fld = $_[0];
+
+ if (defined($fld) && $fld =~ /\S/) {
+ return $fld;
+ } else {
+ return "###";
+ }
+}
+
+# format status output into fixed width columns
+sub status_print {
+ my @fields;
+ my $field;
+ my @flen = ();
+ my $num_fields = 0;
+ my $i;
+ my $pad;
+ my $line;
+
+ # one pass to size the columns
+ foreach $line (@_) {
+ @fields = split(/\s+/, $line);
+ $i = 0;
+ foreach $field (@fields) {
+ if (!defined($flen[$i])) {
+ $flen[$i] = length($field);
+ }
+ else {
+ $flen[$i] = max($flen[$i], length($field));
+ }
+ $i++;
+ }
+
+ if ($i > $num_fields) {
+ $num_fields = $i;
+ }
+ }
+
+ # one pass to print
+ foreach $line (@_) {
+ print " ";
+ @fields = split(/\s+/, $line);
+ for ($i = 0; $i < $num_fields; $i++) {
+ if (defined($fields[$i])) {
+ $pad = $flen[$i] - length($fields[$i]) + 2;
+ }
+ else {
+ $pad = $flen[$i] + 2;
+ }
+ if (defined($fields[$i]) && ($fields[$i] ne "###")) {
+ print "$fields[$i]";
+ }
+ else {
+ print " ";
+ }
+ printf("%*s", $pad, "");
+ }
+ print "\n";
+ }
+}
+
+# check driver load status
+sub check_module_status {
+ if (-e $sys) {
+ return 0;
+ } else {
+ return 1;
+ }
+}
+
+# print driver load status and ethertype for rdma_rxe and rdma_rxe_net
+sub show_module_status {
+ print "rdma_rxe module not loaded\n" if (!(-e $sys));
+}
+
+# print rxe status
+sub do_status {
+ my $instance = $_[0];
+ my $ln = 0;
+ my @outp;
+ my $rxe;
+ my $rmtu;
+
+ get_names();
+ get_dev_info();
+ show_module_status();
+
+ $outp[$ln++] = "Name\tLink\tDriver\t\tSpeed\tNMTU\tIPv4_addr\tRDEV\tRMTU";
+
+ foreach my $eth (@eth_list) {
+
+ # handle case where rxe_drivers are not loaded
+ if (defined($rxe_names{$eth})) {
+ $rxe = $rxe_names{$eth};
+ $rmtu = $rxe_mtu{$rxe};
+ }
+ else {
+ $rxe = "";
+ $rmtu = "";
+ }
+
+ if ((!defined($instance)
+ && (($linkonly == 0) || ($link_state{$eth} =~ "yes")))
+ || (defined($instance) && ($rxe =~ "$instance"))) {
+ $outp[$ln] = set_field("$eth");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$link_state{$eth}");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field(exists($eth_driver{$eth}) ? $eth_driver{$eth} : "");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$link_speed{$eth}");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$eth_mtu{$eth}");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$ipv4_addr{$eth}");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$rxe");
+ $outp[$ln] .= "\t";
+ $outp[$ln] .= set_field("$rmtu");
+ $ln++;
+ }
+ }
+
+ status_print(@outp);
+}
+
+# read file containing list of ethernet devices into a list
+sub populate_persistence {
+ my $i = 0;
+
+ open FILE, $persistence_file;
+ while(<FILE>) {
+ my $line = $_;
+ chomp($line);
+ $line =~ s/^\s+//g;
+ if ($line =~ /[\w]+[\d]/) {
+ # in case we add fields later
+ my ($eth, $cruft) = split(/\s+/, $line, 2);
+ if ($eth =~ /^[\w]+[\d]/) {
+ $persistence_array[$i] = $eth;
+ $persistence_hash{$eth} = $i++;
+ }
+ }
+ }
+ close FILE;
+
+ $num_persistent = $i;
+}
+
+# print out list of ethernet devices to file
+sub commit_persistent {
+ my $i;
+ my $eth;
+
+ open(PF, ">$persistence_file");
+
+ for ($i = 0; $i < $num_persistent; $i++) {
+ $eth = $persistence_array[$i];
+ if ($eth =~ /[\w]+[\d]/) {
+ print(PF "$persistence_array[$i]\n");
+ }
+ }
+
+ close(PF);
+}
+
+sub delete_persistent {
+ my $eth = $_[0];
+
+ if (defined($persistence_hash{$eth})) {
+ $persistence_array[$persistence_hash{$eth}] = "";
+ }
+}
+
+sub add_persistent {
+ my $eth = $_[0];
+
+ # Is this one already in the persistence list?
+ if (!defined($persistence_hash{$eth})) {
+ $persistence_array[$num_persistent] = $eth;
+ $persistence_hash{$eth} = $num_persistent;
+ $num_persistent++;
+ }
+}
+
+# add new rxe device to eth if not already up
+sub rxe_add {
+ my $eth = $_[0];
+
+ if (!($eth =~ /[\w]+[\d]/)) {
+ print "eth_name ($eth) looks bogus\n";
+ return;
+ }
+
+ if (!defined($rxe_names{$eth})) {
+ system("echo '$eth' > $parms/add");
+ }
+ if (!$no_persist) {
+ add_persistent($eth);
+ commit_persistent();
+ }
+}
+
+sub rxe_remove {
+ my $arg2 = $_[0];
+ my $rxe;
+ my $eth;
+
+ print "remove $arg2\n" if ($debug > 0);
+
+ if ($arg2 =~ /[\w]+[\d]/) {
+ $eth = $arg2;
+ $rxe = $rxe_names{$eth};
+ }
+ elsif ($arg2 =~ /rxe[0123456789]/) {
+ $rxe = $arg2;
+ $eth = $eth_names{$rxe};
+ }
+ elsif ($arg2 eq "all") {
+ $rxe = "all";
+ }
+
+ if (($rxe eq "all") || ($rxe =~ /^rxe[0123456789]/)) {
+ my $cmd = "echo '$rxe' > $parms/remove";
+ #print "$cmd\n";
+ system($cmd);
+ if (!$no_persist) {
+ if ($rxe eq "all") {
+ unlink($persistence_file);
+ }
+ elsif ($eth =~/[\w]+[\d]/) {
+ delete_persistent($eth);
+ commit_persistent();
+ }
+ else {
+ print "Warning: Unable to resolve ethname; "
+ . "instance may persist on restart\n";
+ }
+ }
+ }
+ else {
+ print "rxe instance $rxe not found\n";
+ }
+}
+
+sub get_devinfo {
+ my $rxe = $_[0];
+
+ my $cmd = "ibv_devinfo -d $rxe";
+ return `$cmd`;
+}
+
+# allow unsupported modules to load in SLES11 if allowed
+sub modprobe {
+ my $module = $_[0];
+ my $opts = $_[1];
+ my @lines;
+ my $line;
+
+ if ($modprobe_checked == "0") {
+ @lines = `modprobe -c`;
+ foreach $line (@lines) {
+ if ($line =~ /^allow_unsupported_modules *0/) {
+ $modprobe_opt = " --allow-unsupported-modules ";
+ last;
+ }
+ }
+ $modprobe_checked = "1";
+ }
+
+ if (!defined($opts)) {
+ $opts = "";
+ }
+
+ system("modprobe $modprobe_opt $module $opts");
+}
+
+# bring up rxe
+sub do_start {
+ my $proto_str = "";
+
+ system("mkdir -p $persistence_path");
+ system("touch $persistence_file");
+
+ modprobe("ib_core");
+ modprobe("ib_uverbs");
+ modprobe("rdma_ucm");
+ modprobe("rdma_rxe");
+
+ populate_persistence();
+ system("udevadm control --reload");
+
+ foreach my $eth (@persistence_array) {
+ rxe_add($eth);
+ }
+
+ get_names();
+
+ foreach my $rxe (@rxe_array) {
+ my $stat = get_devinfo($rxe);
+ if ($stat =~ "PORT_DOWN") {
+ my $cmd = "ip link set $eth_names{$rxe} up";
+ system($cmd);
+ }
+ }
+
+}
+
+# check if argument is an integer
+sub is_integer {
+ defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
+}
+
+# remove all rxe devices and unload drivers
+sub do_stop {
+ my $rxe;
+
+ foreach $rxe (@rxe_array) {
+ system("echo '$rxe' > $sys/remove");
+ }
+
+ if (-e $sys) {
+ system("rmmod rdma_rxe");
+ }
+
+ if (-e $sys) {
+ print "unable to unload drivers, reboot required\n";
+ }
+}
+
+sub do_debug {
+ my $arg2 = $_[0];
+ my $debugfile = "$parms/debug";
+ chomp($arg2);
+
+ if (!(-e "$debugfile")) {
+ print "Error: debug is compiled out of this rxe driver\n";
+ return;
+ }
+
+ if ($arg2 eq "on") { system("echo '31' > $debugfile"); }
+ elsif ($arg2 eq "off") { system("echo '0' > $debugfile"); }
+ elsif ($arg2 eq "0") { system("echo '0' > $debugfile"); }
+ elsif ($arg2 eq "") { }
+ elsif ($arg2 ge "0" && $arg2 le "31") {
+ system("echo '$arg2' > $debugfile");
+ }
+ else {
+ print "unrecognized debug cmd ($arg2)\n";
+ }
+
+ my $current = read_file($debugfile);
+ chomp($current);
+ if ($current > 0) {
+ print "Debug is ON ($current)\n";
+ }
+ elsif ($current == 0) {
+ print "Debug is OFF\n";
+ }
+ else {
+ print "Unrecognized debug value\n";
+ }
+}
+
+sub max {
+ my $a = $_[0];
+ my $b = $_[1];
+ return $a if ($a > $b);
+ return $b;
+}
+
+# show usage for rxe_cfg
+sub usage {
+ print " Usage:\n";
+ print " rxe_cfg [options] start|stop|status|persistent\n";
+ print " rxe_cfg debug on|off|<num>\n";
+ print " rxe_cfg [-n] add <ndev>\n";
+ print " rxe_cfg [-n] remove <ndev>|<rdev>\n";
+ print "\n";
+ print " <ndev> = network device e.g. eth3\n";
+ print " <rdev> = rdma device e.g. rxe1\n";
+ print "\n";
+ print " Options:\n";
+ print " -h: print this usage information\n";
+ print " -n: do not make the configuration action persistent\n";
+ print " -v: print additional debug output\n";
+ print " -l: show status for interfaces with link up\n";
+ print " -p <num>: (start command only) - set ethertype\n";
+}
+
+sub main {
+ GetOptions(
+ "-h" => \$help,
+ "--help" => \$help,
+ "-n" => \$no_persist,
+ "-v:+" => \$debug,
+ "-f" => \$force,
+ "-l" => \$linkonly,
+ );
+
+ my $arg1 = $ARGV[0];
+ my $arg2 = $ARGV[1];
+ my $arg3 = $ARGV[2];
+
+ # status is the default
+ if (!defined($arg1) || ($arg1 =~ /status/)) {
+ do_status($arg2);
+ exit;
+ }
+
+ if ($help) {
+ usage();
+ exit;
+ }
+
+ # stuff that does not require modules to be loaded
+ if ($arg1 eq "help") { usage(); exit; }
+ elsif ($arg1 eq "start") { do_start(); do_status(); exit; }
+ elsif ($arg1 eq "persistent") { system("cat $persistence_file"); exit; }
+
+
+ # can't do much else, bail if modules aren't loaded
+ if (check_module_status()) {
+ exit;
+ }
+
+ # create persistence file if necessary
+ make_path($persistence_path);
+ if (!(-e $persistence_file)) {
+ `touch $persistence_file`;
+ }
+
+ # Get full context of the configuration
+ populate_persistence();
+ get_names();
+ get_dev_info();
+
+ # Stuff that requires the rdma_rxe module to be loaded
+ if ($arg1 eq "stop") { do_stop(); exit; }
+ elsif ($arg1 eq "debug") { do_debug($arg2); exit; }
+ elsif ($arg1 eq "add") { rxe_add($arg2); exit; }
+ elsif ($arg1 eq "remove") { rxe_remove($arg2); exit; }
+ elsif ($arg1 eq "help") { usage(); exit; }
+}
+
+main();
+
+exit;
diff --git redhat/rdma-core.spec redhat/rdma-core.spec
index 0d4da2ba9503..466433134317 100644
--- redhat/rdma-core.spec
+++ redhat/rdma-core.spec
@@ -428,10 +428,12 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh
%{_libexecdir}/rdma-set-sriov-vf
%{_libexecdir}/mlx4-setup.sh
%{_libexecdir}/truescale-serdes.cmds
+%{_bindir}/rxe_cfg
%{_sbindir}/rdma-ndd
%{_unitdir}/rdma-ndd.service
%{_mandir}/man7/rxe*
%{_mandir}/man8/rdma-ndd.*
+%{_mandir}/man8/rxe*
%license COPYING.*
%files devel
diff --git suse/rdma-core.spec suse/rdma-core.spec
index e49b07abc34a..566aed0497ef 100644
--- suse/rdma-core.spec
+++ suse/rdma-core.spec
@@ -198,7 +198,8 @@ Requires: %{efa_lname} = %{version}-%{release}
Requires: %{mlx4_lname} = %{version}-%{release}
Requires: %{mlx5_lname} = %{version}-%{release}
%endif
-# Recommended packages for rxe
+# Recommended packages for rxe_cfg
+Recommends: ethtool
Recommends: iproute2
%description -n libibverbs
@@ -654,7 +655,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh
%doc %{_docdir}/%{name}-%{version}/libibverbs.md
%doc %{_docdir}/%{name}-%{version}/rxe.md
%doc %{_docdir}/%{name}-%{version}/tag_matching.md
+%{_bindir}/rxe_cfg
%{_mandir}/man7/rxe*
+%{_mandir}/man8/rxe*
%files -n libibnetdisc%{ibnetdisc_major}
%defattr(-, root, root)
