external/cde
1
0
Fork 0
mirror of git://git.code.sf.net/p/cdesktopenv/code synced 2025-03-09 15:50:02 +00:00
Code Issues Releases Wiki Activity

tooltalk: Fix bad assumptions about sizeof(uid_t)

Browse source 
In part of the tooltalk rpc code (mp_message.c), it was assumed that on
the majority of platforms, sizeof(uid_t)=sizeof(gid_t)=sizeof(long).  On
Linux-x64, uid_t is an unsigned int, which makes the code fail: all
tooltalk messages fail to send with an RPC_CANTENCODEARGS at the
rpc-level, and TT_INTERNAL_ERR for the actual program.  We instead
change the code to explicitly examine sizeof(uid_t) to see whether it is
int or long sized. This allows tooltalk-dependent functinoality
like logout and multiple calls to dtfile to work.
This commit is contained in:
Frederic Koehler 2012-08-12 02:12:28 -04:00 • committed by Jon Trulson
parent 19c7c80aa3
commit b33cf9fb60
1 changed files with 28 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

45
cde/lib/tt/lib/mp/mp_message.C
View file

@ -47,7 +47,7 @@
#include <mp/mp_procid.h>
#include <mp/mp_session.h>
#include <util/tt_enumname.h>
#include <util/tt_port.h>
//
// Base constructor for a message. ::xdr() relies on this
@ -302,33 +302,44 @@ xdr(XDR *xdrs)
// Even though in Solaris 2.x uid_t and gid_t are typedef'd as
// u_long, the compiler complains if they are not explicitly
// coerced.
/*
* Obviously there is no xdr_u_uid_t, so we have to hack around this.
* At least by using sizeof checks, we aren't manually specifying
* the size of uid_t depending on the platform (which broke terribly
* when we tried to port to 64-bit linux at first).
*
* A side-effect of testing this way is that we have to spell out the pointer
* casts, since one of them is actually wrong for any given platform (but
* is also a dead code branch)
*/
if (_ptr_guards&_TT_MSK_UID) {
if (xdrs->x_op == XDR_DECODE) {
_uid = 0;
}
#ifdef __osf__
if (! xdr_u_int(xdrs, &_uid)) {
return(0);
if (sizeof (int) == sizeof(uid_t)) {
if (! xdr_u_int(xdrs,(u_int *) &_uid)) return 0;
} else if (sizeof (long) == sizeof(uid_t)) {
if (! xdr_u_long(xdrs,(u_long *) &_gid)) return 0;
} else {
_tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): uid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" );
}
#else
if (! xdr_u_long(xdrs,(u_long *) &_uid)) {
return(0);
}
#endif
}
if (_ptr_guards&_TT_MSK_GID) {
if (xdrs->x_op == XDR_DECODE) {
_gid = 0;
}
#ifdef __osf__
if (! xdr_u_int(xdrs, &_gid)) {
return(0);
if (sizeof (int) == sizeof(gid_t)) {
if (! xdr_u_int(xdrs,(u_int *) &_gid)) {
return(0);
}
} else if (sizeof (long) == sizeof(gid_t)) {
if (! xdr_u_long(xdrs,(u_long *) &_gid)) {
return(0);
}
} else {
_tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): gid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" );
}
#else
if (! xdr_u_long(xdrs,(u_long *) &_gid)) {
return(0);
}
#endif
}
if (_ptr_guards&_TT_MSK_SESSION) {